diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 2336baf..0000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[bumpversion] -current_version = 1.0.6 -commit = True -tag = True - -[bumpversion:file:setup.py] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4c8e15..8946fed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,99 +1,71 @@ +name: CI + on: push: - # Sequence of patterns matched against refs/tags - tags: - - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + branches: [master] + pull_request: + release: + types: [published] + workflow_dispatch: -name: Release +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build: - name: Creating Windows and OSX wheels - + build_wheels: + name: Wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14] steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v7 - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install wheel twine - pip install -e '.' + - name: Build and test wheels + uses: pypa/cibuildwheel@v4.1.0 - - name: Publishing - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py bdist_wheel - twine upload dist/* - - build_manylinux2010: - name: Creating manylinux releases + - uses: actions/upload-artifact@v7 + with: + name: wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl + build_sdist: + name: Source distribution runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - name: Build (manylinux2014) - uses: RalfG/python-wheels-manylinux-build@v0.3-manylinux2014_x86_64 - with: - python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' + - name: Install uv + uses: astral-sh/setup-uv@v8.3.2 + with: + enable-cache: true - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 + - name: Build sdist + run: uv build --sdist - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install wheel twine - - - name: Publishing - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload dist/*-manylinux*.whl - - sdist: - name: Creating source release + - uses: actions/upload-artifact@v7 + with: + name: sdist + path: ./dist/*.tar.gz + publish: + name: Publish to PyPI + needs: [build_wheels, build_sdist] runs-on: ubuntu-latest - + if: github.event_name == 'release' && github.repository == 'TkTech/mutf8' + environment: + name: pypi + url: https://pypi.org/p/mutf8 + permissions: + id-token: write steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install wheel twine + - uses: actions/download-artifact@v8 + with: + path: dist + merge-multiple: true - - name: Publishing - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist - twine upload dist/* + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7ce825f..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Tests - -on: [push] - -jobs: - build: - name: Running tests. - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - - steps: - # Python needs to be setup before checkout to prevent files from being - # left in the source tree. See setup-python/issues/106. - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/checkout@v2 - - - name: Installing platform dependencies - if: matrix.os == 'ubuntu-latest' - run: > - sudo apt-get install -y - build-essential - clang-6.0 - - - name: Installing python dependencies - run: | - python -m pip install --upgrade pip - pip install -e '.[test]' - - - name: Running tests - run: | - pytest diff --git a/.gitignore b/.gitignore index 65b1d76..0e22996 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ *.pyc *.pyd venv +.venv/ .eggs cython_debug/* build/ _build/ +dist/ +wheelhouse/ *.egg-info *.so +uv.lock +*.idea diff --git a/README.md b/README.md index cf6a2ca..5ef4959 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ -![Tests](https://github.com/TkTech/mutf8/workflows/Tests/badge.svg?branch=master) +![CI](https://github.com/TkTech/mutf8/actions/workflows/release.yml/badge.svg) # mutf-8 This package contains simple pure-python as well as C encoders and decoders for -the MUTF-8 character encoding. In most cases, you can also parse the even-rarer -CESU-8. +the MUTF-8 character encoding. These days, you'll most likely encounter MUTF-8 when working on files or protocols related to the JVM. Strings in a Java `.class` file are encoded using @@ -14,26 +13,15 @@ serializer. This library was extracted from [Lawu][], a Python library for working with JVM class files. -## 🎉 Installation +## Installation +Prebuilt binary packages are available for most platforms and Python versions. Install the package from PyPi: ``` pip install mutf8 ``` -Binary wheels are available for the following: - -| | py3.6 | py3.7 | py3.8 | py3.9 | -| ---------------- | ----- | ----- | ----- | ----- | -| OS X (x86_64) | y | y | y | y | -| Windows (x86_64) | y | y | y | y | -| Linux (x86_64) | y | y | y | y | - -If binary wheels are not available, it will attempt to build the C extension -from source with any C99 compiler. If it could not build, it will fall back -to a pure-python version. - ## Usage Encoding and decoding is simple: @@ -48,23 +36,48 @@ bytes = encode_modified_utf8(unicode) This module *does not* register itself globally as a codec, since importing should be side-effect-free. -## 📈 Benchmarks +## Benchmarks -The C extension is significantly faster - often 20x to 40x faster. +The C extension is dramatically faster than the pure-Python fallback — from +~10× on tiny strings (where per-call overhead dominates) to several hundred +times on realistic payloads, and over 1000× when encoding large ASCII. ### MUTF-8 Decoding -| Name | Min (μs) | Max (μs) | StdDev | Ops | -|------------------------------|------------|------------|----------|---------------| -| cmutf8-decode_modified_utf8 | 0.00009 | 0.00080 | 0.00000 | 9957678.56358 | -| pymutf8-decode_modified_utf8 | 0.00190 | 0.06040 | 0.00000 | 450455.96019 | + +| Input | C | Python | Speedup | +|-------|---|--------|--------:| +| ASCII (16 B) | 81.5 ± 26 ns | 1.04 ± 0.18 µs | 13× | +| ASCII (1 KB) | 185 ± 38 ns | 65.7 ± 8.5 µs | 356× | +| ASCII (64 KB) | 5.58 ± 0.66 µs | 4.35 ± 0.23 ms | 780× | +| CJK (16 B) | 127 ± 15 ns | 1.33 ± 0.15 µs | 10× | +| CJK (1 KB) | 681 ± 117 ns | 99.3 ± 9.4 µs | 146× | +| CJK (64 KB) | 31.8 ± 3.3 µs | 6.28 ± 0.063 ms | 198× | +| Emoji (16 B) | 100 ± 21 ns | 882 ± 122 ns | 9× | +| Emoji (1 KB) | 538 ± 119 ns | 79.6 ± 6.9 µs | 148× | +| Emoji (64 KB) | 26.2 ± 4.1 µs | 5.32 ± 0.13 ms | 203× | +| Mixed (16 B) | 118 ± 27 ns | 2.17 ± 0.16 µs | 18× | +| Mixed (1 KB) | 860 ± 162 ns | 76.7 ± 9.2 µs | 89× | +| Mixed (64 KB) | 38 ± 5 µs | 5.3 ± 0.48 ms | 139× | ### MUTF-8 Encoding -| Name | Min (μs) | Max (μs) | StdDev | Ops | -|------------------------------|------------|------------|----------|----------------| -| cmutf8-encode_modified_utf8 | 0.00008 | 0.00151 | 0.00000 | 11897361.05101 | -| pymutf8-encode_modified_utf8 | 0.00180 | 0.16650 | 0.00000 | 474390.98091 | + +| Input | C | Python | Speedup | +|-------|---|--------|--------:| +| ASCII (16 B) | 70.5 ± 5.8 ns | 1.23 ± 0.15 µs | 18× | +| ASCII (1 KB) | 109 ± 16 ns | 49 ± 6.2 µs | 449× | +| ASCII (64 KB) | 1.55 ± 0.27 µs | 2.93 ± 0.19 ms | 1889× | +| CJK (16 B) | 81.4 ± 12 ns | 2.02 ± 0.26 µs | 25× | +| CJK (1 KB) | 624 ± 127 ns | 108 ± 16 µs | 173× | +| CJK (64 KB) | 32.5 ± 2.9 µs | 6.8 ± 0.24 ms | 209× | +| Emoji (16 B) | 73 ± 10 ns | 1.17 ± 0.12 µs | 16× | +| Emoji (1 KB) | 621 ± 76 ns | 67 ± 8.5 µs | 108× | +| Emoji (64 KB) | 33.3 ± 6 µs | 3.98 ± 0.12 ms | 119× | +| Mixed (16 B) | 108 ± 97 ns | 2.48 ± 0.35 µs | 23× | +| Mixed (1 KB) | 1.39 ± 0.29 µs | 63.3 ± 3.5 µs | 45× | +| Mixed (64 KB) | 81.2 ± 14 µs | 3.91 ± 0.093 ms | 48× | + ## C Extension diff --git a/mutf8/__init__.py b/mutf8/__init__.py index 943dc4d..426515b 100644 --- a/mutf8/__init__.py +++ b/mutf8/__init__.py @@ -10,6 +10,8 @@ for MUTF-8/CESU-8 into the python core. """ +__version__ = "1.0.6" + try: from mutf8.cmutf8 import decode_modified_utf8, encode_modified_utf8 diff --git a/mutf8/cmutf8.c b/mutf8/cmutf8.c index e05ddf3..374ec74 100644 --- a/mutf8/cmutf8.c +++ b/mutf8/cmutf8.c @@ -1,6 +1,7 @@ #define PY_SSIZE_T_CLEAN #include #include +#include PyDoc_STRVAR(decode_doc, "Decodes a bytestring containing MUTF-8 as defined in section\n" @@ -32,6 +33,22 @@ decode_modified_utf8(PyObject *self, PyObject *args) // MUTF-8 input. uint8_t *buf = (uint8_t *)view.buf; + + // Fast path: without 0xED (leads every surrogate/6-byte form), 0xC0 (the + // NUL escape) or a raw 0x00 (illegal), the input is plain UTF-8. Offload + // to CPython's optimized decoder; on any error fall back to the strict + // loop below so the exception is still reported as mutf-8. + if (memchr(buf, 0xED, view.len) == NULL && + memchr(buf, 0xC0, view.len) == NULL && + memchr(buf, 0x00, view.len) == NULL) { + PyObject *fast = PyUnicode_DecodeUTF8((const char *)buf, view.len, NULL); + if (fast != NULL) { + PyBuffer_Release(&view); + return fast; + } + PyErr_Clear(); + } + // Array of temporary UCS-4 codepoints. // There's no point using PyUnicode_new and _WriteChar, because // it requires us to have iterated the string to get the maximum unicode @@ -74,33 +91,30 @@ decode_modified_utf8(PyObject *self, PyObject *args) uint8_t b2 = buf[ix + 1]; uint8_t b3 = buf[ix + 2]; - if (x == 0xED && (b2 & 0xF0) == 0xA0) { - if (ix + 5 >= view.len) { - return_err( - "6-byte codepoint started, but input too short" - " to finish."); - } - - // Possible six-byte codepoint. - uint8_t b4 = buf[ix + 3]; + if (x == 0xED && (b2 & 0xF0) == 0xA0 && ix + 5 < view.len && + buf[ix + 3] == 0xED && (buf[ix + 4] & 0xF0) == 0xB0) { + // Six-byte codepoint: a supplementary character written as a + // UTF-16 surrogate pair, each half encoded as its own 3-byte + // sequence. We only take this branch once a low surrogate is + // confirmed to follow; otherwise `x b2 b3` is a lone + // (unpaired) surrogate and is decoded as a normal 3-byte + // codepoint below. uint8_t b5 = buf[ix + 4]; uint8_t b6 = buf[ix + 5]; - - if (b4 == 0xED && (b5 & 0xF0) == 0xB0) { - // Definite six-byte codepoint. - x = ( - 0x10000 | - (b2 & 0x0F) << 0x10 | - (b3 & 0x3F) << 0x0A | - (b5 & 0x0F) << 0x06 | - (b6 & 0x3F) - ); - ix += 5; - cp_out[cp_count++] = x; - continue; - } + x = ( + 0x10000 + + ((b2 & 0x0F) << 0x10 | + (b3 & 0x3F) << 0x0A | + (b5 & 0x0F) << 0x06 | + (b6 & 0x3F)) + ); + ix += 5; + cp_out[cp_count++] = x; + continue; } + // Regular three-byte codepoint. This also covers lone/unpaired + // surrogates, each of which is encoded as a single 3-byte sequence. x = ( (x & 0x0F) << 0x0C | (b2 & 0x3F) << 0x06 | @@ -183,11 +197,23 @@ encode_modified_utf8(PyObject *self, PyObject *args) void *data = PyUnicode_DATA(src); Py_ssize_t length = PyUnicode_GET_LENGTH(src); int kind = PyUnicode_KIND(src); - char *byte_out = PyMem_Calloc(_encoded_size(data, length, kind), 1); - if (!byte_out) { - return PyErr_NoMemory(); + // Fast path: with no supplementary char (kind < 4-byte) and no embedded + // NUL, MUTF-8 == UTF-8 with "surrogatepass" (same 3-byte form for lone + // surrogates). Offload to CPython's optimized encoder. + if (kind != PyUnicode_4BYTE_KIND && + PyUnicode_FindChar(src, 0, 0, length, 1) == -1) { + return PyUnicode_AsEncodedString(src, "utf-8", "surrogatepass"); + } + + // Allocate the result up front at its exact size and write straight into + // it, avoiding a temporary buffer, its zeroing, and a full-output copy. + Py_ssize_t size = _encoded_size(data, length, kind); + PyObject *out = PyBytes_FromStringAndSize(NULL, size); + if (out == NULL) { + return NULL; } + char *byte_out = PyBytes_AS_STRING(out); Py_ssize_t byte_count = 0; @@ -214,7 +240,11 @@ encode_modified_utf8(PyObject *self, PyObject *args) byte_out[byte_count++] = (0x80 | (0x3F & cp)); } else { - // "Two-times-three" byte codepoint. + // Six-byte codepoint: a supplementary character written as a + // UTF-16 surrogate pair, each half encoded as its own 3-byte + // sequence. The codepoint is first offset by 0x10000 as required + // by the surrogate-pair algorithm. + cp -= 0x10000; byte_out[byte_count++] = 0xED; byte_out[byte_count++] = 0xA0 | ((cp >> 0x10) & 0x0F); byte_out[byte_count++] = 0x80 | ((cp >> 0x0A) & 0x3F); @@ -224,8 +254,6 @@ encode_modified_utf8(PyObject *self, PyObject *args) } } - PyObject *out = PyBytes_FromStringAndSize(byte_out, byte_count); - PyMem_Free(byte_out); return out; } diff --git a/mutf8/mutf8.py b/mutf8/mutf8.py index ceec8f5..a154a87 100644 --- a/mutf8/mutf8.py +++ b/mutf8/mutf8.py @@ -59,36 +59,30 @@ def decode_modified_utf8(s: bytes) -> str: b2 = s[s_ix] b3 = s[s_ix + 1] - if b1 == 0xED and (b2 & 0xF0) == 0xA0: - # Possible six-byte codepoint. - if s_ix + 4 >= s_len: - raise UnicodeDecodeError( - 'mutf-8', - s, - s_ix - 1, - s_ix, - '3-byte or 6-byte codepoint started, but input too' - ' short to finish.' - ) - - b4 = s[s_ix + 2] + if (b1 == 0xED and (b2 & 0xF0) == 0xA0 and s_ix + 4 < s_len + and s[s_ix + 2] == 0xED and (s[s_ix + 3] & 0xF0) == 0xB0): + # Six-byte codepoint: a supplementary character written as a + # UTF-16 surrogate pair, each half encoded as its own 3-byte + # sequence. Only take this branch once we've confirmed a low + # surrogate actually follows; otherwise `b1 b2 b3` is a lone + # (unpaired) surrogate and is decoded as a normal 3-byte + # codepoint below. b5 = s[s_ix + 3] b6 = s[s_ix + 4] - - if b4 == 0xED and (b5 & 0xF0) == 0xB0: - # Definite six-byte codepoint. - s_out.append( - chr( - 0x10000 | - (b2 & 0x0F) << 0x10 | - (b3 & 0x3F) << 0x0A | - (b5 & 0x0F) << 0x06 | - (b6 & 0x3F) - ) + s_out.append( + chr( + 0x10000 + + ((b2 & 0x0F) << 0x10 | + (b3 & 0x3F) << 0x0A | + (b5 & 0x0F) << 0x06 | + (b6 & 0x3F)) ) - s_ix += 5 - continue + ) + s_ix += 5 + continue + # Regular three-byte codepoint. This also covers lone/unpaired + # surrogates, each of which is encoded as a single 3-byte sequence. s_out.append( chr( (b1 & 0x0F) << 0x0C | @@ -134,14 +128,18 @@ def encode_modified_utf8(u: str) -> bytes: (0x80 | (0x3F & c)) ]) else: - # Six-byte codepoint. + # Six-byte codepoint: a supplementary character written as a + # UTF-16 surrogate pair, each half encoded as its own 3-byte + # sequence. The codepoint is first offset by 0x10000 as required + # by the surrogate-pair algorithm. + c -= 0x10000 final_string.extend([ 0xED, 0xA0 | ((c >> 0x10) & 0x0F), - 0x80 | ((c >> 0x0A) & 0x3f), + 0x80 | ((c >> 0x0A) & 0x3F), 0xED, - 0xb0 | ((c >> 0x06) & 0x0f), - 0x80 | (c & 0x3f) + 0xB0 | ((c >> 0x06) & 0x0F), + 0x80 | (c & 0x3F) ]) return bytes(final_string) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6b83b19 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "mutf8" +dynamic = ["version"] +description = "Fast MUTF-8 encoder & decoder" +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENCE"] +authors = [{ name = "Tyler Kennedy", email = "tk@tkte.ch" }] +keywords = ["mutf-8", "cesu-8", "jvm"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: C", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[project.optional-dependencies] +test = ["pytest", "pytest-benchmark"] + +[project.urls] +Homepage = "https://github.com/TkTech/mutf8" +Source = "https://github.com/TkTech/mutf8" +Issues = "https://github.com/TkTech/mutf8/issues" + +[tool.setuptools] +packages = ["mutf8"] +# optional=true so a failed compile falls back to the pure-Python mutf8/mutf8.py. +ext-modules = [ + { name = "mutf8.cmutf8", sources = ["mutf8/cmutf8.c"], language = "c", optional = true }, +] + +[tool.setuptools.dynamic] +version = { attr = "mutf8.__version__" } + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = ["slow: mark test as slow to run"] + +[tool.cibuildwheel] +skip = ["*_i686"] +test-extras = ["test"] +test-command = "pytest {project}/tests" + +[tool.bumpversion] +current_version = "1.0.6" +commit = true +tag = true +tag_name = "v{new_version}" +allow_dirty = false + +[[tool.bumpversion.files]] +filename = "mutf8/__init__.py" diff --git a/setup.py b/setup.py deleted file mode 100644 index 04c32b8..0000000 --- a/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import os.path - -from setuptools import setup, find_packages, Extension - - -root = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(root, 'README.md'), 'rb') as readme: - long_description = readme.read().decode('utf-8') - - -setup( - name='mutf8', - packages=find_packages(), - version='1.0.6', - description='Fast MUTF-8 encoder & decoder', - long_description=long_description, - long_description_content_type='text/markdown', - author='Tyler Kennedy', - author_email='tk@tkte.ch', - url='http://github.com/TkTech/mutf8', - keywords=['mutf-8', 'cesu-8', 'jvm'], - classifiers=[ - 'Programming Language :: Python :: 3', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Intended Audience :: Developers', - ], - extras_require={ - 'test': [ - 'pytest', - 'pytest-benchmark' - ] - }, - ext_modules=[ - Extension( - 'mutf8.cmutf8', - ['mutf8/cmutf8.c'], - language='c', - optional=True - ) - ] -) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index e86bec0..656aa51 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,28 +1,67 @@ +""" +Benchmarks comparing the pure-Python and C encoders/decoders across realistic +inputs (ASCII, CJK, emoji, mixed) and payload sizes. + +Marked ``slow``; run with ``pytest --runslow tests/test_benchmark.py``. +""" import pytest import mutf8.mutf8 as pymutf8 import mutf8.cmutf8 as cmutf8 +# Short representative units, repeated to hit each target size. Ordered from +# the most common real-world case (ASCII JVM symbols) to the rarest. +_UNITS = { + "ascii": "java/lang/Object;", # 1-byte codepoints + "cjk": "編碼測試字", # 3-byte codepoints + "emoji": "🚀🎉", # 6-byte (surrogate pair) codepoints + "mixed": "café ☕ item#42 shipped 🚚 ", # 1/2/3/6-byte mix +} + +# Target *encoded* sizes: 16b exposes per-call overhead, the rest throughput. +_SIZES = {"16b": 16, "1kb": 1024, "64kb": 64 * 1024} + + +def _scale(unit, target_bytes): + reps = max(1, round(target_bytes / len(cmutf8.encode_modified_utf8(unit)))) + return unit * reps + + +# (content, size) -> decoded str and its MUTF-8 encoding. +STR_CORPORA = { + (content, size): _scale(unit, nbytes) + for content, unit in _UNITS.items() + for size, nbytes in _SIZES.items() +} +BYTES_CORPORA = { + key: cmutf8.encode_modified_utf8(text) for key, text in STR_CORPORA.items() +} + +CORPORA = list(STR_CORPORA) +_CORPUS_IDS = [f"{content}-{size}" for content, size in CORPORA] +_IMPLS = (("py", pymutf8), ("c", cmutf8)) +_IMPL_IDS = [name for name, _ in _IMPLS] + -@pytest.mark.parametrize('group,decoder', ( - ('pymutf8', pymutf8.decode_modified_utf8), - ('cmutf8', cmutf8.decode_modified_utf8) -)) @pytest.mark.slow -def test_decode(group, decoder, benchmark): - """Compare the performance of the python and C decoders.""" - benchmark.group = 'MUTF-8 Decoding' - benchmark.extra_info['group'] = group - benchmark(decoder, b'\xED\xA0\xBD\xED\xB8\x88') +@pytest.mark.parametrize("corpus", CORPORA, ids=_CORPUS_IDS) +@pytest.mark.parametrize("impl,module", _IMPLS, ids=_IMPL_IDS) +def test_decode(corpus, impl, module, benchmark): + content, size = corpus + data = BYTES_CORPORA[corpus] + benchmark.group = f"decode {content}-{size}" + benchmark.extra_info.update(impl=impl, encoded_bytes=len(data)) + result = benchmark(module.decode_modified_utf8, data) + assert result == STR_CORPORA[corpus] -@pytest.mark.parametrize('group,encoder', ( - ('pymutf8', pymutf8.encode_modified_utf8), - ('cmutf8', cmutf8.encode_modified_utf8) -)) @pytest.mark.slow -def test_encode(group, encoder, benchmark): - """Compare the performance of the python and C encoders.""" - benchmark.group = 'MUTF-8 Encoding' - benchmark.extra_info['group'] = group - benchmark(encoder, '\U0001F608') +@pytest.mark.parametrize("corpus", CORPORA, ids=_CORPUS_IDS) +@pytest.mark.parametrize("impl,module", _IMPLS, ids=_IMPL_IDS) +def test_encode(corpus, impl, module, benchmark): + content, size = corpus + text = STR_CORPORA[corpus] + benchmark.group = f"encode {content}-{size}" + benchmark.extra_info.update(impl=impl, encoded_bytes=len(BYTES_CORPORA[corpus])) + result = benchmark(module.encode_modified_utf8, text) + assert result == BYTES_CORPORA[corpus] diff --git a/tests/test_bugs.py b/tests/test_bugs.py index 9b07775..7dafa71 100644 --- a/tests/test_bugs.py +++ b/tests/test_bugs.py @@ -32,3 +32,61 @@ def test_issue_3(encoder, decoder): """ str = '黑人抬棺組裝包' assert decoder(encoder(str)) == str + + +# Supplementary characters and their *correct* MUTF-8 encoding, cross-checked +# against Java's DataOutputStream.writeUTF, Android/DEX and Rust's +# residua-mutf8. See https://github.com/TkTech/mutf8/issues/5 and /issues/6. +ISSUE_5_6_VECTORS = ( + ('\U00010000', b'\xed\xa0\x80\xed\xb0\x80'), + ('\U00020000', b'\xed\xa1\x80\xed\xb0\x80'), + ('\U00040000', b'\xed\xa3\x80\xed\xb0\x80'), + ('\U00080000', b'\xed\xa7\x80\xed\xb0\x80'), + ('\U00100000', b'\xed\xaf\x80\xed\xb0\x80'), + ('\U0010FFFF', b'\xed\xaf\xbf\xed\xbf\xbf'), + ('\U00010401', b'\xed\xa0\x81\xed\xb0\x81'), # residua-mutf8 example + ('\U0001F4CB', b'\xed\xa0\xbd\xed\xb3\x8b'), # 📋 + ('\U0001F3F9', b'\xed\xa0\xbc\xed\xbf\xb9'), # 🏹 + ('\U0001F41F', b'\xed\xa0\xbd\xed\xb0\x9f'), # 🐟 +) + + +def test_issue_5_and_6(encoder, decoder): + """ + Supplementary characters (> U+FFFF) must be encoded as a UTF-16 surrogate + pair after subtracting 0x10000, matching Java/Android/CESU-8. Previously + the 0x10000 offset was omitted on encode (and compensated with a bitwise + OR on decode), producing non-standard bytes and losing data for plane 16. + + https://github.com/TkTech/mutf8/issues/5 + https://github.com/TkTech/mutf8/issues/6 + """ + for text, expected in ISSUE_5_6_VECTORS: + assert encoder(text) == expected, hex(ord(text)) + # Decoding standard MUTF-8 must recover the original codepoint. + assert decoder(expected) == text, hex(ord(text)) + # And a full round-trip must be lossless across the whole range. + assert decoder(encoder(text)) == text, hex(ord(text)) + + +def test_issue_4(decoder, encoder): + """ + A lone (unpaired) high surrogate is legal in a Java string and is encoded + as a single 3-byte sequence (0xED 0xA0-0xAF ...). The decoder must not + assume every 0xED 0xAx sequence begins a six-byte pair; when no low + surrogate follows (including at end-of-input) it must decode a 3-byte + lone surrogate instead of raising "input too short". + + https://github.com/TkTech/mutf8/issues/4 + """ + # A lone high surrogate at end-of-input (previously raised spuriously). + assert decoder(b'\xed\xa1\x80') == '\ud840' + # A lone low surrogate, likewise. + assert decoder(b'\xed\xb0\x80') == '\udc00' + # A high surrogate followed by non-surrogate data, not a pair. + assert decoder(b'\xed\xa1\x80AB') == '\ud840AB' + # Two high surrogates in a row: each is its own 3-byte codepoint. + assert decoder(b'\xed\xa1\x80\xed\xa1\x80') == '\ud840\ud840' + # Lone surrogates must survive a round-trip. + for cp in (0xD800, 0xD840, 0xDBFF, 0xDC00, 0xDFFF): + assert decoder(encoder(chr(cp))) == chr(cp), hex(cp) diff --git a/tests/test_modified_utf8.py b/tests/test_modified_utf8.py index 0a51773..32b18bb 100644 --- a/tests/test_modified_utf8.py +++ b/tests/test_modified_utf8.py @@ -125,44 +125,44 @@ def test_six_byte(module): Test data taken from py2jdbc. """ pairs = _pairs( - (0x10000, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x80)), - (0x10001, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x81)), - (0x10002, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x82)), - (0x10004, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x84)), - (0x10008, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x88)), - (0x10010, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x90)), - (0x10020, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0xa0)), - (0x10040, (0xed, 0xa1, 0x80, 0xed, 0xb1, 0x80)), - (0x10080, (0xed, 0xa1, 0x80, 0xed, 0xb2, 0x80)), - (0x10100, (0xed, 0xa1, 0x80, 0xed, 0xb4, 0x80)), - (0x10200, (0xed, 0xa1, 0x80, 0xed, 0xb8, 0x80)), - (0x10400, (0xed, 0xa1, 0x81, 0xed, 0xb0, 0x80)), - (0x10800, (0xed, 0xa1, 0x82, 0xed, 0xb0, 0x80)), - (0x11000, (0xed, 0xa1, 0x84, 0xed, 0xb0, 0x80)), - (0x12000, (0xed, 0xa1, 0x88, 0xed, 0xb0, 0x80)), - (0x14000, (0xed, 0xa1, 0x90, 0xed, 0xb0, 0x80)), - (0x18000, (0xed, 0xa1, 0xa0, 0xed, 0xb0, 0x80)), - (0x30000, (0xed, 0xa3, 0x80, 0xed, 0xb0, 0x80)), - (0x50000, (0xed, 0xa5, 0x80, 0xed, 0xb0, 0x80)), - (0x90000, (0xed, 0xa9, 0x80, 0xed, 0xb0, 0x80)), - (0x10003, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x83)), - (0x10007, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x87)), - (0x1000f, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x8f)), - (0x1001f, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0x9f)), - (0x1003f, (0xed, 0xa1, 0x80, 0xed, 0xb0, 0xbf)), - (0x1007f, (0xed, 0xa1, 0x80, 0xed, 0xb1, 0xbf)), - (0x100ff, (0xed, 0xa1, 0x80, 0xed, 0xb3, 0xbf)), - (0x101ff, (0xed, 0xa1, 0x80, 0xed, 0xb7, 0xbf)), - (0x103ff, (0xed, 0xa1, 0x80, 0xed, 0xbf, 0xbf)), - (0x107ff, (0xed, 0xa1, 0x81, 0xed, 0xbf, 0xbf)), - (0x10fff, (0xed, 0xa1, 0x83, 0xed, 0xbf, 0xbf)), - (0x11fff, (0xed, 0xa1, 0x87, 0xed, 0xbf, 0xbf)), - (0x13fff, (0xed, 0xa1, 0x8f, 0xed, 0xbf, 0xbf)), - (0x17fff, (0xed, 0xa1, 0x9f, 0xed, 0xbf, 0xbf)), - (0x1ffff, (0xed, 0xa1, 0xbf, 0xed, 0xbf, 0xbf)), - (0x3ffff, (0xed, 0xa3, 0xbf, 0xed, 0xbf, 0xbf)), - (0x7ffff, (0xed, 0xa7, 0xbf, 0xed, 0xbf, 0xbf)), - (0xfffff, (0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf)) + (0x10000, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80)), + (0x10001, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x81)), + (0x10002, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x82)), + (0x10004, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x84)), + (0x10008, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x88)), + (0x10010, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x90)), + (0x10020, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0xa0)), + (0x10040, (0xed, 0xa0, 0x80, 0xed, 0xb1, 0x80)), + (0x10080, (0xed, 0xa0, 0x80, 0xed, 0xb2, 0x80)), + (0x10100, (0xed, 0xa0, 0x80, 0xed, 0xb4, 0x80)), + (0x10200, (0xed, 0xa0, 0x80, 0xed, 0xb8, 0x80)), + (0x10400, (0xed, 0xa0, 0x81, 0xed, 0xb0, 0x80)), + (0x10800, (0xed, 0xa0, 0x82, 0xed, 0xb0, 0x80)), + (0x11000, (0xed, 0xa0, 0x84, 0xed, 0xb0, 0x80)), + (0x12000, (0xed, 0xa0, 0x88, 0xed, 0xb0, 0x80)), + (0x14000, (0xed, 0xa0, 0x90, 0xed, 0xb0, 0x80)), + (0x18000, (0xed, 0xa0, 0xa0, 0xed, 0xb0, 0x80)), + (0x30000, (0xed, 0xa2, 0x80, 0xed, 0xb0, 0x80)), + (0x50000, (0xed, 0xa4, 0x80, 0xed, 0xb0, 0x80)), + (0x90000, (0xed, 0xa8, 0x80, 0xed, 0xb0, 0x80)), + (0x10003, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x83)), + (0x10007, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x87)), + (0x1000f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x8f)), + (0x1001f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0x9f)), + (0x1003f, (0xed, 0xa0, 0x80, 0xed, 0xb0, 0xbf)), + (0x1007f, (0xed, 0xa0, 0x80, 0xed, 0xb1, 0xbf)), + (0x100ff, (0xed, 0xa0, 0x80, 0xed, 0xb3, 0xbf)), + (0x101ff, (0xed, 0xa0, 0x80, 0xed, 0xb7, 0xbf)), + (0x103ff, (0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf)), + (0x107ff, (0xed, 0xa0, 0x81, 0xed, 0xbf, 0xbf)), + (0x10fff, (0xed, 0xa0, 0x83, 0xed, 0xbf, 0xbf)), + (0x11fff, (0xed, 0xa0, 0x87, 0xed, 0xbf, 0xbf)), + (0x13fff, (0xed, 0xa0, 0x8f, 0xed, 0xbf, 0xbf)), + (0x17fff, (0xed, 0xa0, 0x9f, 0xed, 0xbf, 0xbf)), + (0x1ffff, (0xed, 0xa0, 0xbf, 0xed, 0xbf, 0xbf)), + (0x3ffff, (0xed, 0xa2, 0xbf, 0xed, 0xbf, 0xbf)), + (0x7ffff, (0xed, 0xa6, 0xbf, 0xed, 0xbf, 0xbf)), + (0xfffff, (0xed, 0xae, 0xbf, 0xed, 0xbf, 0xbf)) ) for decoded, original in pairs: