From acf4dfe97397e49f2b844fcc8051439fcfee4e73 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Tue, 21 Oct 2025 13:33:08 -0400 Subject: [PATCH 1/7] Add python 3.13 support - Add Python 3.13 to Github Actions workflow matrices - Update CIBW_PROJECT_REQUIRES - Add classifier to pyproject.toml - Update ray dependency with python 3.13 constraint - Fix InMemoryCatalog to close connections - Add pytest filter to ignore rest of 3.13 SQLite Resource Warnings --- .github/workflows/pypi-build-artifacts.yml | 3 ++- .github/workflows/python-ci.yml | 2 +- .github/workflows/svn-build-artifacts.yml | 3 ++- pyproject.toml | 8 +++++++- tests/catalog/test_base.py | 8 +++++--- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pypi-build-artifacts.yml b/.github/workflows/pypi-build-artifacts.yml index 65de2a7828..56d8107685 100644 --- a/.github/workflows/pypi-build-artifacts.yml +++ b/.github/workflows/pypi-build-artifacts.yml @@ -45,6 +45,7 @@ jobs: 3.10 3.11 3.12 + 3.13 - name: Install poetry run: make install-poetry @@ -68,7 +69,7 @@ jobs: env: # Ignore 32 bit architectures CIBW_ARCHS: "auto64" - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10" + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<=3.13" CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1" CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py" # Ignore tests for pypy since not all dependencies are compiled for it diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index c8e19c6441..b3b5cb3db2 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.10', '3.11', '3.12'] + python: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/svn-build-artifacts.yml b/.github/workflows/svn-build-artifacts.yml index 116c6538f4..36fac96eb6 100644 --- a/.github/workflows/svn-build-artifacts.yml +++ b/.github/workflows/svn-build-artifacts.yml @@ -45,6 +45,7 @@ jobs: 3.10 3.11 3.12 + 3.13 - name: Install poetry run: make install-poetry @@ -63,7 +64,7 @@ jobs: env: # Ignore 32 bit architectures CIBW_ARCHS: "auto64" - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10" + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<=3.13" CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1" CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py" # Ignore tests for pypy since not all dependencies are compiled for it diff --git a/pyproject.toml b/pyproject.toml index 5b1292b0b2..f6736785f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] packages = [ { include = "pyiceberg" }, @@ -66,7 +67,10 @@ pyarrow = { version = ">=17.0.0", optional = true } google-auth = { version = ">=2.4.0", optional = true } pandas = { version = ">=1.0.0,<3.0.0", optional = true } duckdb = { version = ">=0.5.0,<2.0.0", optional = true } -ray = { version = ">=2.10.0,<=2.44.0", optional = true } +ray = [ + { version = ">=2.10.0,<3.0.0", python = ">=3.10", optional = true }, + { version = ">=2.45.0", python = ">=3.13", optional = true } +] python-snappy = { version = ">=0.6.0,<1.0.0", optional = true } thrift = { version = ">=0.13.0,<1.0.0", optional = true } boto3 = { version = ">=1.24.59", optional = true } @@ -343,6 +347,8 @@ markers = [ # Turns a warning into an error filterwarnings = [ "error", + # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections + "ignore:unclosed database in InMemoryCatalog: - return InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"}) +def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: + catalog = InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"}) + yield catalog + catalog.close() TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table") From 2ad4a5b760befd04c7f2a6355f092ac2a1dbf8ec Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 24 Dec 2025 12:09:22 -0800 Subject: [PATCH 2/7] add more 3.13 --- .github/workflows/python-ci.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index e909640f28..e7299d3fe8 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -71,7 +71,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.10', '3.11', '3.12'] + python: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v6 diff --git a/pyproject.toml b/pyproject.toml index 1c856cd12a..9ba0ab54d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] dependencies = [ "mmh3>=4.0.0,<6.0.0", From ba69fb2f7b187243b6503a916ffccaa196c54dd3 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 24 Dec 2025 12:12:53 -0800 Subject: [PATCH 3/7] fix typing --- tests/catalog/test_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index bb2a32bae3..96e04c198a 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -17,6 +17,7 @@ # pylint:disable=redefined-outer-name +from collections.abc import Generator from pathlib import PosixPath import pyarrow as pa @@ -51,7 +52,7 @@ @pytest.fixture -def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: +def catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]: catalog = InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"}) yield catalog catalog.close() From f43a4fbdf13393c85601934907afd9ebfaf63ad3 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 24 Dec 2025 12:20:56 -0800 Subject: [PATCH 4/7] upgrade ray to version with 3.13 support --- pyproject.toml | 2 +- uv.lock | 36 +++++++++++++++++------------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ba0ab54d8..f7ce99013c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ duckdb = [ "pyarrow>=17.0.0", ] ray = [ - "ray>=2.10.0,<=2.44.0", + "ray>=2.10.0,<3.0.0", "pyarrow>=17.0.0", "pandas>=1.0.0,<3.0.0", ] diff --git a/uv.lock b/uv.lock index 64a759d19d..79e480c1b6 100644 --- a/uv.lock +++ b/uv.lock @@ -3846,7 +3846,7 @@ requires-dist = [ { name = "pyparsing", specifier = ">=3.1.0,<4.0.0" }, { name = "pyroaring", specifier = ">=1.0.0,<2.0.0" }, { name = "python-snappy", marker = "extra == 'snappy'", specifier = ">=0.6.0,<1.0.0" }, - { name = "ray", marker = "extra == 'ray'", specifier = ">=2.10.0,<=2.44.0" }, + { name = "ray", marker = "extra == 'ray'", specifier = ">=2.10.0,<3.0.0" }, { name = "requests", specifier = ">=2.20.0,<3.0.0" }, { name = "rich", specifier = ">=10.11.0,<15.0.0" }, { name = "s3fs", marker = "extra == 's3fs'", specifier = ">=2023.1.0" }, @@ -4225,13 +4225,11 @@ wheels = [ [[package]] name = "ray" -version = "2.44.0" +version = "2.53.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiosignal" }, { name = "click" }, { name = "filelock" }, - { name = "frozenlist" }, { name = "jsonschema" }, { name = "msgpack" }, { name = "packaging" }, @@ -4240,21 +4238,21 @@ dependencies = [ { name = "requests" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/63/d6/bee5993f7e4a5de8937047b86e5b6cb486b953a57a67274acf028587e583/ray-2.44.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:632790c327e6931a7a8ccadde8fd3afaeb73ad382f87df4dd47a52ca8bfe051c", size = 68183659, upload-time = "2025-03-21T05:22:57.18Z" }, - { url = "https://files.pythonhosted.org/packages/68/a0/446c510cb613f67a0592522a8e356cd9e1c0cdf4015d4760bc8e2e8d5bfe/ray-2.44.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5dfbf26b30aec37e5d4425c660145e5520299a8855324686e2f17fc8601bf4c8", size = 65482853, upload-time = "2025-03-21T05:23:04.801Z" }, - { url = "https://files.pythonhosted.org/packages/c9/3e/e01fc118a9a77d91d57da0718d8eeaac336034640b53e416c5c576c7b46a/ray-2.44.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a99fbb9ad2c1af221870d86b8a8e3d59c18a5513adde9d7088a4a275dc59da7f", size = 67031809, upload-time = "2025-03-21T05:23:10.944Z" }, - { url = "https://files.pythonhosted.org/packages/bb/81/d17f50b0c995850232df17a3bfc314c462e27bffa990c63a25a021f03855/ray-2.44.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:b4fdbd4b2b5b45e413dc16a19a4abf5000d36c3c5854908dca4697323ff5d7e1", size = 67948459, upload-time = "2025-03-21T05:23:16.965Z" }, - { url = "https://files.pythonhosted.org/packages/37/c3/29c94bc274120b666292e07299039a130bf54c351eba0415b2b1aae1ddd8/ray-2.44.0-cp310-cp310-win_amd64.whl", hash = "sha256:0258a48e49f531f83a7c65c3482df0c6568491e35ac606a6f651fb4718e49dbb", size = 25746862, upload-time = "2025-03-21T05:23:22.252Z" }, - { url = "https://files.pythonhosted.org/packages/11/1c/850cb2ee30444cfbec4cc69d0ae49b54a2bd3dbd6b896535b855f5a5e2a3/ray-2.44.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:fbe4832cb2efcfc0493ea4742b4828b1eb0dabcfedf87f64be6be1d0ce874c69", size = 68146593, upload-time = "2025-03-21T05:23:28.326Z" }, - { url = "https://files.pythonhosted.org/packages/a9/7a/f4cc6da9953b620f7a550850f389bf971ba53820786a47f580ce2e384d3e/ray-2.44.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5337227dc9f8084280c29456988a244ca9b4ce0fbc7385d73070120f47e46979", size = 65444864, upload-time = "2025-03-21T05:23:34.591Z" }, - { url = "https://files.pythonhosted.org/packages/7b/67/c1bf9bce604b5c234f7c2180a851be0dde2a01134183f065b69ea74a0d07/ray-2.44.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:c337237e7a8a1d8702dcf67e0a98ea8cd4ec0357d288bf0816f8990c258d8bc3", size = 67171926, upload-time = "2025-03-21T05:23:40.637Z" }, - { url = "https://files.pythonhosted.org/packages/ee/5c/ae371cb53bf7413043629f64278a5156c8588ab10fbd7ac72aa7806d0d16/ray-2.44.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:864f0a69b3cd7ca4eb7043f7f79dc9ce8b71a2c982eeec7f117f48f2846b713c", size = 68075001, upload-time = "2025-03-21T05:23:47.093Z" }, - { url = "https://files.pythonhosted.org/packages/35/16/5c280ced4f3f18c30ba69ee9cf5642dd1ede379943bdb520c606f3d7970a/ray-2.44.0-cp311-cp311-win_amd64.whl", hash = "sha256:18cef276f2789a3ed22d78274d58803e28defb66ff8d03bdce1ea8b433dea5f8", size = 25702733, upload-time = "2025-03-21T05:23:52.396Z" }, - { url = "https://files.pythonhosted.org/packages/f9/2f/15ec0da8b0716b838299760b88b7ffc042ef79d40248a799364eb2ef1b13/ray-2.44.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:53dc75ea2b4fd869ea4a6cca9de5e02aa24f2f0d18e0a08b8a765ab2be65dd1c", size = 68138830, upload-time = "2025-03-21T05:23:57.54Z" }, - { url = "https://files.pythonhosted.org/packages/30/64/77d2839d66058530ca434dd0d5fa4299480644208f1966cdd972f3ada7f6/ray-2.44.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:398e9be193c97f734af019f0eface1f45c94195b96ecc4a647ad607650df572c", size = 65431753, upload-time = "2025-03-21T05:24:04.94Z" }, - { url = "https://files.pythonhosted.org/packages/34/5b/fa9a467d2b8b53d04be1743cd7732a5ef98f5227200bb75544cdae0c75eb/ray-2.44.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:2a911e699e483ac4879110b608b06b35e602191c0e7b97326ca497c5caafe6a8", size = 67189648, upload-time = "2025-03-21T05:24:11.206Z" }, - { url = "https://files.pythonhosted.org/packages/70/eb/aba1de11b1dda0257599ed04260c9fec9cb3d0e77f4abcac9319db59e60d/ray-2.44.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:0d65ac523801e40a397bbf552f406867bb9469dd261046ca63cdc2ec3110db87", size = 68129021, upload-time = "2025-03-21T05:24:17.539Z" }, - { url = "https://files.pythonhosted.org/packages/98/b9/04043b1fa05994b459eb32cd8fcebe495df5558099a705e6dd1e99b76444/ray-2.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:9bb3b6df352653c4479325161a0e17e2c6b3278661c69ff842602d7440312af7", size = 25682480, upload-time = "2025-03-21T05:24:22.865Z" }, + { url = "https://files.pythonhosted.org/packages/2f/99/21986c7f8135dafbf7c49229c52faaa9d2d365db7d86fffe978dde8ee967/ray-2.53.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4db914a0a6dd608fa49c066929a1282745a2dbd73caee67d7b80fe684ca65bdd", size = 69473649, upload-time = "2025-12-20T16:05:40.58Z" }, + { url = "https://files.pythonhosted.org/packages/70/d9/58b5426a3f11993851db3c93841358cebdddd948153481d355b720f31f9d/ray-2.53.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4108280d8a1cb90d7d68e5c954c35e63b8bb9a4ba15f88c5e7da0e2025647712", size = 71342662, upload-time = "2025-12-20T16:05:46.936Z" }, + { url = "https://files.pythonhosted.org/packages/c5/05/4aa32370b313481c2d1d41cb53ec786daebdb2ef665b01ef2ac43d9cf457/ray-2.53.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:4dbb5fce1364763f29741055f50abe33cf726397141f9cc0e845dd3cc963e455", size = 72188620, upload-time = "2025-12-20T16:05:52.817Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c6/21efe5886898421df20078a333b0984eade7d7aa4bdc68a336f0c66db27e/ray-2.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:90faf630d20b6abf3135997fb3edb5842134aff92e04ee709865db04816d97ef", size = 27200553, upload-time = "2025-12-20T16:05:57.655Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/d5c29a4b014d8b9a624203a88b67630072c1d6960425dbf7a1f0fa5d6b74/ray-2.53.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:bd3ec4c342776ddac23ae2b108c64f5939f417ccc4875900d586c7c978463269", size = 69479296, upload-time = "2025-12-20T16:06:05.111Z" }, + { url = "https://files.pythonhosted.org/packages/c6/41/9e19d1e5d9458a5ba157c36642e2874bcb22fddbd7c1e77b668e5afc3f3d/ray-2.53.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:a0bbb98b0b0f25a3ee075ca10171e1260e70b6bc690cd509ecd7ce1228af854d", size = 71463449, upload-time = "2025-12-20T16:06:10.983Z" }, + { url = "https://files.pythonhosted.org/packages/63/de/58c19906b0dd16ea06b4f2465b7327f5f180e6b6e1c8c9b610d7c589ea5f/ray-2.53.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:eb000c17f7301071fdd15c44c4cd3ac0f7953bb4c7c227e61719fe7048195bcd", size = 72305102, upload-time = "2025-12-20T16:06:17.989Z" }, + { url = "https://files.pythonhosted.org/packages/b1/43/72cc1cfe17d26abe62a793eab10445f9546dce24192b85a6cd0cdc47ed86/ray-2.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:4a1bb3fe09ab4cd0d16ddc96b9f60c9ed83b3f93b87aa8506e0d3b746fd4e825", size = 27194174, upload-time = "2025-12-20T16:06:23.042Z" }, + { url = "https://files.pythonhosted.org/packages/b2/44/562718a634e63e8ef7985285288a167d4af62bc2a7decce3300cf937776a/ray-2.53.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:d8b95d047d947493803fb8417aea31225dcacdab15afdc75b8a238901949d457", size = 69463763, upload-time = "2025-12-20T16:06:28.685Z" }, + { url = "https://files.pythonhosted.org/packages/38/68/8e59b8413f3751fe7ce8b98ee8787d13964b47a4043587950790a9dd2151/ray-2.53.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:65e2ce58d3dc6baa3cf45824d889c1968ebde565ee54dfd80a98af8f31af8e4a", size = 71504450, upload-time = "2025-12-20T16:06:34.922Z" }, + { url = "https://files.pythonhosted.org/packages/2a/db/978a50d264565ca42e2a4bf115ec9a1f04f19ca5e620e6aa2f280747b644/ray-2.53.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:14f46363e9b4cf0c1c8b4d8623ec337c5bd408377831b5e5b50067930137bbca", size = 72370424, upload-time = "2025-12-20T16:06:40.821Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6c/bba6f22a9d83ee8f236000ba315f0c197bdc79888b4fa42fd762f729cbbd/ray-2.53.0-cp312-cp312-win_amd64.whl", hash = "sha256:b828c147f9ff2f277b1d254e4fe9a746fdfaee7e313a93a97c7edf4dae9b81a4", size = 27178106, upload-time = "2025-12-20T16:06:45.594Z" }, + { url = "https://files.pythonhosted.org/packages/3d/38/450cf9cf3c490fa4cc6d470597f819444da60f85579d2b34b95ee79fcb6f/ray-2.53.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:85b472ab6fb8f1189f8cef81913fd91b24dd69b3fa7dcca7e144827bd924f6c0", size = 69409819, upload-time = "2025-12-20T16:06:50.668Z" }, + { url = "https://files.pythonhosted.org/packages/71/5e/d452970b07174d5e4f8688abae889d01321b51ced827db1f1d1cb7d56d44/ray-2.53.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:7196e5358dfcc8211be864f45e6dfe4827202df294af3c7a76ff8fbc080e0522", size = 71409529, upload-time = "2025-12-20T16:06:56.2Z" }, + { url = "https://files.pythonhosted.org/packages/cb/84/50b317a125617a638a64694c12f56183edd5df01828a35fa4c55c7b13c66/ray-2.53.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:73dbbaa7962a7f5e38aa8cf9483e0e9817205e989aa3dc859c738c2af1ae01df", size = 72283961, upload-time = "2025-12-20T16:07:05.831Z" }, ] [[package]] From c6e6420a85be6a67ab14035a4c70f4ca93d21fac Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 24 Dec 2025 14:16:25 -0800 Subject: [PATCH 5/7] fix ray local init --- pyproject.toml | 3 +++ tests/conftest.py | 13 +++++++++++++ tests/integration/test_reads.py | 9 +++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7ce99013c..b5692c2229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,6 +163,9 @@ filterwarnings = [ "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core", # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections "ignore:unclosed database in Generator[Any, None, None]: + """Fixture to manage Ray initialization and shutdown for tests.""" + import ray + + ray.init( + ignore_reinit_error=True, + runtime_env={"working_dir": None}, # Prevent Ray from serializing the working directory to workers + ) + yield ray + ray.shutdown() diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index 785037aef3..cd98399efe 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -21,6 +21,7 @@ import uuid from datetime import datetime, timedelta from pathlib import PosixPath +from typing import Any from urllib.parse import urlparse import pyarrow as pa @@ -357,7 +358,7 @@ def test_bodo_nan(catalog: Catalog, monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.integration @pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) -def test_ray_nan(catalog: Catalog) -> None: +def test_ray_nan(catalog: Catalog, ray_session: Any) -> None: table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten") ray_dataset = table_test_null_nan_rewritten.scan().to_ray() assert ray_dataset.count() == 3 @@ -366,7 +367,7 @@ def test_ray_nan(catalog: Catalog) -> None: @pytest.mark.integration @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) -def test_ray_nan_rewritten(catalog: Catalog) -> None: +def test_ray_nan_rewritten(catalog: Catalog, ray_session: Any) -> None: table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten") ray_dataset = table_test_null_nan_rewritten.scan( row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric") @@ -379,7 +380,7 @@ def test_ray_nan_rewritten(catalog: Catalog) -> None: @pytest.mark.integration @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) @pytest.mark.skip(reason="Fixing issues with NaN's: https://github.com/apache/arrow/issues/34162") -def test_ray_not_nan_count(catalog: Catalog) -> None: +def test_ray_not_nan_count(catalog: Catalog, ray_session: Any) -> None: table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten") ray_dataset = table_test_null_nan_rewritten.scan(row_filter=NotNaN("col_numeric"), selected_fields=("idx",)).to_ray() assert ray_dataset.count() == 2 @@ -387,7 +388,7 @@ def test_ray_not_nan_count(catalog: Catalog) -> None: @pytest.mark.integration @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) -def test_ray_all_types(catalog: Catalog) -> None: +def test_ray_all_types(catalog: Catalog, ray_session: Any) -> None: table_test_all_types = catalog.load_table("default.test_all_types") ray_dataset = table_test_all_types.scan().to_ray() pandas_dataframe = table_test_all_types.scan().to_pandas() From c7121b4685724eb2355cca76327da7a88f1a7786 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 24 Dec 2025 16:39:26 -0800 Subject: [PATCH 6/7] share the same local ray cluster --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index b9ecaae39f..9dff42589d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2949,7 +2949,7 @@ def pyarrow_table_with_promoted_types(pyarrow_schema_with_promoted_types: "pa.Sc ) -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def ray_session() -> Generator[Any, None, None]: """Fixture to manage Ray initialization and shutdown for tests.""" import ray From 5efec28e854d0d7338e85fda0959f894cffb35db Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 24 Dec 2025 17:20:18 -0800 Subject: [PATCH 7/7] ignore just the ray tests --- pyproject.toml | 3 --- tests/integration/test_reads.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b5692c2229..f7ce99013c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,9 +163,6 @@ filterwarnings = [ "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core", # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections "ignore:unclosed database in None: @pytest.mark.integration +@pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) def test_ray_nan_rewritten(catalog: Catalog, ray_session: Any) -> None: table_test_null_nan_rewritten = catalog.load_table("default.test_null_nan_rewritten") @@ -378,6 +379,7 @@ def test_ray_nan_rewritten(catalog: Catalog, ray_session: Any) -> None: @pytest.mark.integration +@pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) @pytest.mark.skip(reason="Fixing issues with NaN's: https://github.com/apache/arrow/issues/34162") def test_ray_not_nan_count(catalog: Catalog, ray_session: Any) -> None: @@ -387,6 +389,7 @@ def test_ray_not_nan_count(catalog: Catalog, ray_session: Any) -> None: @pytest.mark.integration +@pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) def test_ray_all_types(catalog: Catalog, ray_session: Any) -> None: table_test_all_types = catalog.load_table("default.test_all_types")