Merge branch 'main' into datafusion_53

Author: timsaucer

.github/workflows/build.yml

@@ -98,6 +98,15 @@ jobs:
       - name: Check Cargo.toml formatting
         run: taplo format --check
 
+  check-crates-patch:
+    if: inputs.build_mode == 'release'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Ensure [patch.crates-io] is empty
+        run: python3 dev/check_crates_patch.py
+
   generate-license:
     runs-on: ubuntu-latest
     steps:

.github/workflows/test.yml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ~/.cargo
-          key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }}
+          key: cargo-cache-${{ matrix.toolchain }}-${{ hashFiles('Cargo.lock') }}
 
       - name: Install dependencies
         uses: astral-sh/setup-uv@v7
@@ -106,7 +106,7 @@ jobs:
           RUST_BACKTRACE: 1
         run: |
           git submodule update --init
-          uv run --no-project pytest -v . --import-mode=importlib
+          uv run --no-project pytest -v --import-mode=importlib
 
       - name: FFI unit tests
         run: |

Cargo.toml

@@ -92,7 +92,8 @@ crate-type = ["cdylib", "rlib"]
 lto = true
 codegen-units = 1
 
-# TODO: remove when datafusion-53 is released
+# We cannot publish to crates.io with any patches in the below section. Developers
+# must remove any entries in this section before creating a release candidate.
 [patch.crates-io]
 datafusion = { git = "https://github.com/apache/datafusion.git", rev = "518560246e87d489eba6d511fa167aa429b06728" }
 datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "518560246e87d489eba6d511fa167aa429b06728" }

dev/check_crates_patch.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Check that no Cargo.toml files contain [patch.crates-io] entries.
+
+Release builds must not depend on patched crates. During development it is
+common to temporarily patch crates-io dependencies, but those patches must
+be removed before creating a release.
+
+An empty [patch.crates-io] section is allowed.
+"""
+
+import sys
+from pathlib import Path
+
+import tomllib
+
+
+def main() -> int:
+    errors: list[str] = []
+    for cargo_toml in sorted(Path().rglob("Cargo.toml")):
+        if "target" in cargo_toml.parts:
+            continue
+        with Path.open(cargo_toml, "rb") as f:
+            data = tomllib.load(f)
+        patch = data.get("patch", {}).get("crates-io", {})
+        if patch:
+            errors.append(str(cargo_toml))
+            for name, spec in patch.items():
+                errors.append(f"  {name} = {spec}")
+
+    if errors:
+        print("ERROR: Release builds must not contain [patch.crates-io] entries.")
+        print()
+        for line in errors:
+            print(line)
+        print()
+        print("Remove all [patch.crates-io] entries before creating a release.")
+        return 1
+
+    print("OK: No [patch.crates-io] entries found.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pyproject.toml

@@ -70,6 +70,9 @@ features = ["substrait"]
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
+addopts = "--doctest-modules"
+doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
+testpaths = ["python/tests", "python/datafusion"]
 
 # Enable docstring linting using the google style guide
 [tool.ruff.lint]

python/datafusion/dataframe.py

@@ -327,8 +327,9 @@ def into_view(self, temporary: bool = False) -> Table:
             >>> df = ctx.sql("SELECT 1 AS value")
             >>> view = df.into_view()
             >>> ctx.register_table("values_view", view)
-            >>> df.collect()  # The DataFrame is still usable
-            >>> ctx.sql("SELECT value FROM values_view").collect()
+            >>> result = ctx.sql("SELECT value FROM values_view").collect()
+            >>> result[0].column("value").to_pylist()
+            [1]
         """
         from datafusion.catalog import Table as _Table
 
@@ -1379,9 +1380,12 @@ def fill_null(self, value: Any, subset: list[str] | None = None) -> DataFrame:
             DataFrame with null values replaced where type casting is possible
 
         Examples:
-            >>> df = df.fill_null(0)  # Fill all nulls with 0 where possible
-            >>> # Fill nulls in specific string columns
-            >>> df = df.fill_null("missing", subset=["name", "category"])
+            >>> from datafusion import SessionContext, col
+            >>> ctx = SessionContext()
+            >>> df = ctx.from_pydict({"a": [1, None, 3], "b": [None, 5, 6]})
+            >>> filled = df.fill_null(0)
+            >>> filled.sort(col("a")).collect()[0].column("a").to_pylist()
+            [0, 1, 3]
 
         Notes:
             - Only fills nulls in columns where the value can be cast to the column type