diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d9bcc4..1aeddbc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,11 +18,18 @@ jobs: - "3.12" - "3.13" - "3.14" + - "3.15-dev" extras: - - "[test,docs,zodb]" - # include: - # - python-version: "3.13" - # extras: "[test,docs,gevent,pyramid]" + - "[test,docs,zodb,orjson]" + include: + # orjson 3.11.9 doesn't currently support + # being built for free-threaded python, so if we wanted + # to test there, we'd need to substitute it, most likely by + # moving it to an extra. + - python-version: "3.14t" + extras: "[test,docs,zodb]" + - python-version: "3.15t-dev" + extras: "[test,docs,zodb]" runs-on: ubuntu-latest steps: @@ -38,13 +45,20 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -U coverage python -m pip install -v -U -e ".${{ matrix.extras }}" + - name: Disable GIL + if: ${{endsWith(matrix.python-version, 't') || endsWith(matrix.python-version, 't-dev')}} + # zope.hookable 8.2/zope.proxy 7.2 currently enables the gil + run: | + echo PYTHON_GIL=0 >> $GITHUB_ENV - name: Test run: | python -m coverage run -m zope.testrunner --test-path=src --auto-color --auto-progress PURE_PYTHON=1 coverage run -a -m zope.testrunner --test-path=src --auto-color --auto-progress + - name: Docs + if: matrix.python-version == '3.14' + # Requires orjson + run: | coverage run -a -m sphinx -b doctest -d docs/_build/doctrees docs docs/_build/doctests - coverage combine || true - coverage report -i || true - name: Lint if: matrix.python-version == '3.14' run: | @@ -54,6 +68,10 @@ jobs: run: | python -m pip uninstall -y ZODB zope.dublincore persistent zope.container BTrees PURE_PYTHON=1 coverage run -a -m zope.testrunner --test-path=src --auto-color --auto-progress + - name: Combine Coverage + run: | + coverage combine || true + coverage report -i || true - name: Submit to Coveralls uses: coverallsapp/github-action@v2 with: @@ -75,7 +93,7 @@ jobs: # We use a regular Python matrix entry to share as much code as possible. strategy: matrix: - python-version: [3.12] + python-version: [3.14] image: - manylinux_2_28_x86_64 - manylinux_2_28_aarch64 diff --git a/.readthedocs.yml b/.readthedocs.yml index 11bc75d..c4396a1 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -24,7 +24,7 @@ build: # os is required for some reason os: ubuntu-22.04 tools: - python: "3.11" + python: "3.14" python: install: @@ -32,3 +32,4 @@ python: path: . extra_requirements: - docs + - orjson diff --git a/CHANGES.rst b/CHANGES.rst index 53b5729..361d6ea 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,10 +3,18 @@ ========= -3.1.1 (unreleased) +3.2.0 (unreleased) ================== -- Nothing changed yet. +- Drop support for Python 3.10. +- Add support for Python 3.15. +- Move the ``orjson`` dependency to the ``orjson`` extra + for compatibility with free-threaded Python. If you're using regular + Python, it is highly recommended to install this extra. +- Add support for free-threaded CPython. However, note that some + dependencies, most notably orjson, cannot + currently be installed on free-threaded CPython, and other + dependencies may currently require enabling the GIL. 3.1.0 (2026-05-08) diff --git a/docs/basics.rst b/docs/basics.rst index f3f1e7d..ff9510e 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -60,6 +60,23 @@ concerns should be kept as separated as possible from our model objects. Ideally, we should be able to use third-party objects that we have no control over seamlessly in external and internal data. +Installation +============ + +This package is installed from PyPI:: + + pip install nti.externalization[orjson,zodb] + +It has some extras: + +orjson + Highly recommended for a faster dumping and loading experience. As + of at least orjson 3.11.9, however, this is incompatible with + free-threaded Python. +zodb + Optional; provides support for BTrees, persistent objects, intids, and + container proxies. + Getting Started =============== @@ -248,7 +265,7 @@ others: >>> internal.creator = u'sjohnson' >>> internal.createdTime = 123456 - >>> pprint(to_external_object(internal)) + >>> pprint(to_external_object(internal), compact=True, indent=1) {'Class': 'ExternalObject', 'CreatedTime': 123456, 'Creator': 'sjohnson', @@ -337,7 +354,7 @@ Now we can register and use it as before: ... postal_code=u'95014', ... country=u'USA') >>> external = to_external_object(address) - >>> pprint(external) + >>> pprint(external, compact=True, indent=1) {'Class': 'Address', 'city': 'Cupertino', 'country': 'USA', @@ -473,7 +490,7 @@ demonstrating that nested schemas and objects are possible. ... realname=u'Steve Jobs', ... ) >>> external = to_external_object(user_profile) - >>> pprint(external) + >>> pprint(external, compact=True, indent=1) {'Class': 'UserProfile', 'MimeType': 'application/vnd.nextthought.benchmarks.userprofile', 'addresses': {'home': {'Class': 'Address', diff --git a/docs/conf.py b/docs/conf.py index 8d4ed18..b2b3f9b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -78,7 +78,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/externalization.rst b/docs/externalization.rst index adef431..bd4728f 100644 --- a/docs/externalization.rst +++ b/docs/externalization.rst @@ -85,7 +85,7 @@ We'll register our adapter and externalize: >>> from zope import component >>> component.provideSubscriptionAdapter(PrivateAddressDecorator) >>> from pprint import pprint - >>> pprint(to_external_object(home_address)) + >>> pprint(to_external_object(home_address), compact=True, indent=1) {'Class': 'Address', 'MimeType': 'application/vnd.nextthought.benchmarks.address', 'country': 'USA', @@ -118,7 +118,7 @@ look for a request): :pyversion: > 3.3 >>> component.provideSubscriptionAdapter(LinkAddressDecorator) - >>> pprint(to_external_object(home_address, request=Request())) + >>> pprint(to_external_object(home_address, request=Request()), compact=True, indent=1) {'Class': 'Address', 'MimeType': 'application/vnd.nextthought.benchmarks.address', 'country': 'USA', diff --git a/make-manylinux b/make-manylinux index 4837235..163b1f2 100755 --- a/make-manylinux +++ b/make-manylinux @@ -13,7 +13,7 @@ export TRAVIS=true # know. The env var works for pip 20. export PIP_NO_PYTHON_VERSION_WARNING=1 export PIP_NO_WARN_SCRIPT_LOCATION=1 - +export PIP_ROOT_USER_ACTION=ignore if [ -d /project ] && [ -d /opt/python ]; then # Running inside docker @@ -44,9 +44,9 @@ if [ -d /project ] && [ -d /opt/python ]; then OPATH="$PATH" which auditwheel echo @@@@@@@@@@@@@@@@@@@@@@ - echo Will build /opt/python/cp{310,311,312,313,314}* + echo Will build /opt/python/cp{311,312,313,314,315}* - for variant in `ls -d /opt/python/cp{314,313,310,311,312}*`; do + for variant in `ls -d /opt/python/cp{315,314,313,311,312}*`; do export PATH="$variant/bin:$OPATH" echo "Building $variant $(python --version)" diff --git a/setup.py b/setup.py index 2e49cee..47c239b 100755 --- a/setup.py +++ b/setup.py @@ -133,6 +133,7 @@ def _c(m): 'language_level': '3', 'always_allow_keywords': False, 'nonecheck': False, + 'freethreading_compatible': True, }, ) except ValueError: @@ -147,7 +148,7 @@ def _c(m): setup( name='nti.externalization', - version='3.1.1.dev0', + version='3.2.0.dev0', author='Jason Madden', author_email='jason@seecoresoftware.com', description="NTI Externalization", @@ -160,11 +161,11 @@ def _c(m): 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3.15', 'Programming Language :: Python :: Implementation :: CPython', ], url="https://github.com/OpenNTI/nti.externalization", @@ -179,7 +180,7 @@ def _c(m): 'PyYAML >= 5.1', 'isodate', 'pytz', - 'orjson >= 3.11.9', + 'transaction', 'zope.component >= 4.6.1', 'zope.configuration >= 4.4.0', @@ -212,7 +213,10 @@ def _c(m): 'benchmarks': [ 'pyperf', ], + 'orjson': [ + 'orjson >= 3.11.9', + ] }, entry_points=entry_points, - python_requires=">=3.10", + python_requires=">=3.11", ) diff --git a/src/nti/externalization/representation.py b/src/nti/externalization/representation.py index 85b918f..81d994e 100644 --- a/src/nti/externalization/representation.py +++ b/src/nti/externalization/representation.py @@ -20,7 +20,13 @@ class POSError(Exception): # type:ignore[no-redef] """Mock""" else: from ZODB.POSException import POSError # type:ignore[no-redef] -import orjson +try: + import orjson + _HAS_ORJSON = True +except ModuleNotFoundError: + import json + _HAS_ORJSON = False + import yaml from zope import component from zope import interface @@ -39,6 +45,8 @@ class POSError(Exception): # type:ignore[no-redef] 'to_json_representation_sorted', 'WithRepr', 'JsonRepresenter', + 'OrJsonRepresenter', + 'StdJsonRepresenter', 'YamlRepresenter', ] @@ -102,7 +110,8 @@ def to_json_representation_fast(obj) -> bytes: and additional parameters to optimize for speed. Note that this bypasses utility lookup and directly - uses :class:`JsonRepresenter` + uses :class:`JsonRepresenter`. It is also only + fastest when using orjson. .. versionadded:: 3.0.0 .. versionchanged:: 3.1.0 @@ -148,7 +157,7 @@ def _second_pass_to_external_object(obj): @interface.named(EXT_REPR_JSON) @interface.implementer(IExternalObjectIO) -class JsonRepresenter(object): +class OrJsonRepresenter: """ Default IO object using ``orjson`` for JSON input/output. """ @@ -169,7 +178,9 @@ def dump(obj, fp=None, sort_keys=False, as_str=True, **_unused) -> str|bytes: Added the *sort_keys* parameter, defaulting to false for speed. Added the *as_str* parameter, defaulting to true for backwards compatibility. If set to false, then a bytes object will be returned (and written to any - *fp*). Bytes is orjson's native output format. + *fp*). Bytes is orjson's native output format, meaning no encoding/decoding is + required when this is false. + Other keyword arguments are ignored. @@ -186,6 +197,56 @@ def dump(obj, fp=None, sort_keys=False, as_str=True, **_unused) -> str|bytes: def load(self, stream): return orjson.loads(stream) + +@interface.named(EXT_REPR_JSON) +@interface.implementer(IExternalObjectIO) +class StdJsonRepresenter: + """ + Default IO object using :mod:`json` for JSON input/output. + """ + + @staticmethod + def dump(obj, fp=None, sort_keys=False, as_str=True, **_unused) -> str|bytes: + """ + dump(obj, fp=None, sort_keys=False, as_str=True) -> str|bytes + + Given an object that is known to already be in an externalized form, + convert it to JSON. This can be about 10% faster then requiring a pass + across all the sub-objects of the object to check that they are in external + form, while still handling a few corner cases with a second-pass conversion. + (These things creep in during the object decorator phase and are usually + links.) + + .. versionchanged:: 3.0.0 + Added the *sort_keys* parameter, defaulting to false for speed. + Added the *as_str* parameter, defaulting to true for backwards compatibility + and speed. + If set to false, then a bytes object will be returned (and written to any + *fp*). Because str is the standard library's default output format, this + requires decoding. + + Other keyword arguments are ignored. + + """ + result = json.dumps(obj, # pylint: disable=used-before-assignment + sort_keys=sort_keys, + default=_second_pass_to_external_object) + if not as_str: + result = result.encode('utf-8') # type:ignore[assignment] + if fp: + return fp.write(result) + return result + + def load(self, stream): + return json.loads(stream) + + +if _HAS_ORJSON: + JsonRepresenter = OrJsonRepresenter +else: + JsonRepresenter = StdJsonRepresenter # type:ignore + + # This is meant for dumping already externalized objects, but # because of the second_pass_to_external_object default, # it will actually dump any dumpable object by first externalizing @@ -255,7 +316,7 @@ def construct_yaml_str(self, node): @interface.implementer(IExternalObjectIO) class YamlRepresenter(object): """ - Default IO object using ``yaml`` for object input/output. + Default IO object using :mod:`yaml` for object input/output. """ @staticmethod diff --git a/src/nti/externalization/tests/test_docs.py b/src/nti/externalization/tests/test_docs.py index c51edfe..9f87a89 100644 --- a/src/nti/externalization/tests/test_docs.py +++ b/src/nti/externalization/tests/test_docs.py @@ -37,6 +37,11 @@ manuel.codeblock.CODEBLOCK_START = CODEBLOCK_START def test_suite(): + try: + import orjson + except ModuleNotFoundError: + orjson = None # type:ignore + here = os.path.dirname(__file__) while not os.path.exists(os.path.join(here, 'setup.py')): here = os.path.join(here, '..') @@ -48,7 +53,8 @@ def test_suite(): 'basics.rst', 'externalization.rst', 'internalization.rst', - ) + ) if orjson else () + paths = [os.path.join(docs, f) for f in files_to_test] kwargs = {'tearDown': lambda _: cleanup.cleanUp()} m = manuel.ignore.Manuel() diff --git a/src/nti/externalization/tests/test_externalization.py b/src/nti/externalization/tests/test_externalization.py index d6dad49..d3157f8 100644 --- a/src/nti/externalization/tests/test_externalization.py +++ b/src/nti/externalization/tests/test_externalization.py @@ -832,6 +832,10 @@ def toExternalObject(self, **_kwargs): def test_recursive_call_on_creator(self): # Make sure that we properly handle recursive calls on a # field we want to pre-convert to a str, creator. + try: + __import__('orjson') + except ModuleNotFoundError: + self.skipTest('Requires orjson') class O(object): def __init__(self): diff --git a/src/nti/externalization/tests/test_representation.py b/src/nti/externalization/tests/test_representation.py index 6f56556..2dffaf7 100644 --- a/src/nti/externalization/tests/test_representation.py +++ b/src/nti/externalization/tests/test_representation.py @@ -221,6 +221,10 @@ def _expected_nan_str(self, val): def test_dump_decimal_nan(self): import decimal + try: + __import__('orjson') + except ModuleNotFoundError: + self.skipTest('Requires orjson') # type:ignore rep = self._makeOne() for f in float('-nan'), float('nan'): @@ -235,6 +239,11 @@ def _expected_decimal_inf_val(self, val): def test_dump_decimal_inf(self): import decimal + try: + __import__('orjson') + except ModuleNotFoundError: + self.skipTest('Requires orjson') # type:ignore + rep = self._makeOne() for f in float('-inf'), float('inf'): diff --git a/tox.ini b/tox.ini index 61ad20f..7d112d0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py310,py311,py312,py313,py314,py314-pure,coverage,docs,lint + py311,py312,py313,py314,py314-pure,py314t,py315t,coverage,docs,lint [testenv] usedevelop = true @@ -9,10 +9,12 @@ commands = extras = test zodb + orjson setenv = pure: PURE_PYTHON=1 ZOPE_INTERFACE_STRICT_IRO=1 + [testenv:coverage] usedevelop = true basepython = @@ -30,12 +32,28 @@ setenv = extras = test +[testenv:py314t] +setenv: + PYTHON_GIL=0 +extras = + test + zodb + +[testenv:py315t] +setenv: + PYTHON_GIL=0 +extras = + test + zodb + [testenv:docs] commands = sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html # XXX: Fix doctests #python -m sphinx -b doctest -d docs/_build/doctrees docs docs/_build/doctests -extras = docs +extras = + docs + orjson setenv = PURE_PYTHON = 1