diff --git a/Cargo.lock b/Cargo.lock index f04cde59d6a40..2b45811c48ecc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,54 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "abi_stable" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" -dependencies = [ - "abi_stable_derive", - "abi_stable_shared", - "const_panic", - "core_extensions", - "crossbeam-channel", - "generational-arena", - "libloading", - "lock_api", - "parking_lot", - "paste", - "repr_offset", - "rustc_version", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "abi_stable_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" -dependencies = [ - "abi_stable_shared", - "as_derive_utils", - "core_extensions", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", - "typed-arena", -] - -[[package]] -name = "abi_stable_shared" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" -dependencies = [ - "core_extensions", -] - [[package]] name = "adler2" version = "2.0.1" @@ -485,18 +437,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "as_derive_utils" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" -dependencies = [ - "core_extensions", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "astral-tokio-tar" version = "0.6.0" @@ -530,9 +470,6 @@ name = "async-ffi" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" -dependencies = [ - "abi_stable", -] [[package]] name = "async-recursion" @@ -1469,15 +1406,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "const_panic" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" -dependencies = [ - "typewit", -] - [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1500,21 +1428,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_extensions" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" -dependencies = [ - "core_extensions_proc_macros", -] - -[[package]] -name = "core_extensions_proc_macros" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1570,15 +1483,6 @@ dependencies = [ "itertools 0.13.0", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -2182,7 +2086,6 @@ dependencies = [ name = "datafusion-ffi" version = "52.3.0" dependencies = [ - "abi_stable", "arrow", "arrow-schema", "async-ffi", @@ -2206,9 +2109,11 @@ dependencies = [ "datafusion-session", "doc-comment", "futures", + "libloading", "log", "prost", "semver", + "stabby", "tokio", ] @@ -2945,7 +2850,6 @@ dependencies = [ name = "ffi_example_table_provider" version = "0.1.0" dependencies = [ - "abi_stable", "arrow", "datafusion", "datafusion-ffi", @@ -2956,7 +2860,6 @@ dependencies = [ name = "ffi_module_interface" version = "0.1.0" dependencies = [ - "abi_stable", "datafusion-ffi", ] @@ -2964,10 +2867,10 @@ dependencies = [ name = "ffi_module_loader" version = "0.1.0" dependencies = [ - "abi_stable", "datafusion", "datafusion-ffi", "ffi_module_interface", + "libloading", "tokio", ] @@ -3167,15 +3070,6 @@ dependencies = [ "prost-build", ] -[[package]] -name = "generational-arena" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" -dependencies = [ - "cfg-if", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -3899,12 +3793,12 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "winapi", + "windows-link", ] [[package]] @@ -5138,15 +5032,6 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" -[[package]] -name = "repr_offset" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" -dependencies = [ - "tstr", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -5632,6 +5517,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -5774,6 +5665,41 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "stabby" +version = "72.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976399a0c48ea769ef7f5dc303bb88240ab8d84008647a6b2303eced3dab3945" +dependencies = [ + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "72.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b54832a9a1f92a0e55e74a5c0332744426edc515bb3fbad82f10b874a87f0d" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "72.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a768b1e51e4dbfa4fa52ae5c01241c0a41e2938fdffbb84add0c8238092f9091" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "rand 0.8.5", + "syn 1.0.109", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -6418,45 +6344,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tstr" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" -dependencies = [ - "tstr_proc_macros", -] - -[[package]] -name = "tstr_proc_macros" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" - [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - [[package]] name = "typenum" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" -[[package]] -name = "typewit" -version = "1.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" - [[package]] name = "typify" version = "0.5.0" diff --git a/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml b/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml index e2d0e3fa6744d..3cfa6dcf90f18 100644 --- a/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml +++ b/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml @@ -22,7 +22,6 @@ edition = { workspace = true } publish = false [dependencies] -abi_stable = "0.11.3" arrow = { workspace = true } datafusion = { workspace = true } datafusion-ffi = { workspace = true } diff --git a/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs b/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs index eb217ef9e4832..7894e97f3796d 100644 --- a/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs +++ b/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs @@ -17,13 +17,12 @@ use std::sync::Arc; -use abi_stable::{export_root_module, prefix_type::PrefixTypeTrait}; use arrow::array::RecordBatch; use arrow::datatypes::{DataType, Field, Schema}; use datafusion::{common::record_batch, datasource::MemTable}; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::table_provider::FFI_TableProvider; -use ffi_module_interface::{TableProviderModule, TableProviderModuleRef}; +use ffi_module_interface::TableProviderModule; fn create_record_batch(start_value: i32, num_values: usize) -> RecordBatch { let end_value = start_value + num_values as i32; @@ -56,11 +55,10 @@ extern "C" fn construct_simple_table_provider( FFI_TableProvider::new_with_ffi_codec(Arc::new(table_provider), true, None, codec) } -#[export_root_module] +#[unsafe(no_mangle)] /// This defines the entry point for using the module. -pub fn get_simple_memory_table() -> TableProviderModuleRef { +pub extern "C" fn ffi_example_get_module() -> TableProviderModule { TableProviderModule { create_table: construct_simple_table_provider, } - .leak_into_prefix() } diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml index fe4902711241e..0244cb2a5ed15 100644 --- a/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml +++ b/datafusion-examples/examples/ffi/ffi_module_interface/Cargo.toml @@ -25,5 +25,4 @@ publish = false workspace = true [dependencies] -abi_stable = "0.11.3" datafusion-ffi = { workspace = true } diff --git a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs index 3b2b9e1871dae..54a59c9e5d073 100644 --- a/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs +++ b/datafusion-examples/examples/ffi/ffi_module_interface/src/lib.rs @@ -15,36 +15,17 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::{ - StableAbi, declare_root_module_statics, - library::{LibraryError, RootModule}, - package_version_strings, - sabi_types::VersionStrings, -}; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::table_provider::FFI_TableProvider; -#[repr(C)] -#[derive(StableAbi)] -#[sabi(kind(Prefix(prefix_ref = TableProviderModuleRef)))] /// This struct defines the module interfaces. It is to be shared by /// both the module loading program and library that implements the /// module. It is possible to move this definition into the loading /// program and reference it in the modules, but this example shows /// how a user may wish to separate these concerns. +#[repr(C)] pub struct TableProviderModule { /// Constructs the table provider pub create_table: extern "C" fn(codec: FFI_LogicalExtensionCodec) -> FFI_TableProvider, } - -impl RootModule for TableProviderModuleRef { - declare_root_module_statics! {TableProviderModuleRef} - const BASE_NAME: &'static str = "ffi_example_table_provider"; - const NAME: &'static str = "ffi_example_table_provider"; - const VERSION_STRINGS: VersionStrings = package_version_strings!(); - - fn initialization(self) -> Result { - Ok(self) - } -} diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml index 8d7434dca211b..48821a9310769 100644 --- a/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml +++ b/datafusion-examples/examples/ffi/ffi_module_loader/Cargo.toml @@ -25,8 +25,8 @@ publish = false workspace = true [dependencies] -abi_stable = "0.11.3" datafusion = { workspace = true } datafusion-ffi = { workspace = true } ffi_module_interface = { path = "../ffi_module_interface" } +libloading = "0.8" tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] } diff --git a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs index 8ce5b156df3b1..0657c4a08fa86 100644 --- a/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs +++ b/datafusion-examples/examples/ffi/ffi_module_loader/src/main.rs @@ -18,28 +18,53 @@ use std::sync::Arc; use datafusion::{ + datasource::TableProvider, error::{DataFusionError, Result}, + execution::TaskContextProvider, prelude::SessionContext, }; - -use abi_stable::library::{RootModule, development_utils::compute_library_path}; -use datafusion::datasource::TableProvider; -use datafusion::execution::TaskContextProvider; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; -use ffi_module_interface::TableProviderModuleRef; +use ffi_module_interface::TableProviderModule; #[tokio::main] async fn main() -> Result<()> { // Find the location of the library. This is specific to the build environment, // so you will need to change the approach here based on your use case. - let target: &std::path::Path = "../../../../target/".as_ref(); - let library_path = compute_library_path::(target) - .map_err(|e| DataFusionError::External(Box::new(e)))?; + let lib_prefix = if cfg!(target_os = "windows") { + "" + } else { + "lib" + }; + let lib_ext = if cfg!(target_os = "macos") { + "dylib" + } else if cfg!(target_os = "windows") { + "dll" + } else { + "so" + }; + + let build_type = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + + let library_path = format!( + "../../../../target/{build_type}/{lib_prefix}ffi_example_table_provider.{lib_ext}" + ); + + // Load the library using libloading + let lib = unsafe { + libloading::Library::new(&library_path) + .map_err(|e| DataFusionError::External(Box::new(e)))? + }; + + let get_module: libloading::Symbol TableProviderModule> = unsafe { + lib.get(b"ffi_example_get_module") + .map_err(|e| DataFusionError::External(Box::new(e)))? + }; - // Load the module - let table_provider_module = - TableProviderModuleRef::load_from_directory(&library_path) - .map_err(|e| DataFusionError::External(Box::new(e)))?; + let table_provider_module = get_module(); let ctx = Arc::new(SessionContext::new()); let codec = FFI_LogicalExtensionCodec::new_default( @@ -48,12 +73,7 @@ async fn main() -> Result<()> { // By calling the code below, the table provided will be created within // the module's code. - let ffi_table_provider = - table_provider_module - .create_table() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_table".to_string(), - ))?(codec); + let ffi_table_provider = (table_provider_module.create_table)(codec); // In order to access the table provider within this executable, we need to // turn it into a `TableProvider`. diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.toml index 28e1b2ee5681f..69b4d39192a5f 100644 --- a/datafusion/ffi/Cargo.toml +++ b/datafusion/ffi/Cargo.toml @@ -44,10 +44,9 @@ crate-type = ["cdylib", "rlib"] # It increases build times and library binary size for users. [dependencies] -abi_stable = "0.11.3" arrow = { workspace = true, features = ["ffi"] } arrow-schema = { workspace = true } -async-ffi = { version = "0.5.0", features = ["abi_stable"] } +async-ffi = { version = "0.5.0" } async-trait = { workspace = true } datafusion-catalog = { workspace = true } datafusion-common = { workspace = true } @@ -66,9 +65,11 @@ datafusion-proto = { workspace = true } datafusion-proto-common = { workspace = true } datafusion-session = { workspace = true } futures = { workspace = true } +libloading = "0.8" log = { workspace = true } prost = { workspace = true } semver = "1.0.27" +stabby = "72.1.1" tokio = { workspace = true } [dev-dependencies] diff --git a/datafusion/ffi/README.md b/datafusion/ffi/README.md index 304ebb90f49dd..c7fab1e05bf29 100644 --- a/datafusion/ffi/README.md +++ b/datafusion/ffi/README.md @@ -65,7 +65,7 @@ to work across Rust libraries. In general, you can use Rust's [FFI] to operate across different programming languages, but that is not the design intent of this crate. Instead, we are using external crates that provide stable interfaces that closely mirror the Rust native approach. To learn more -about this approach see the [abi_stable] and [async-ffi] crates. +about this approach see the [stabby] and [async-ffi] crates. If you have a library in another language that you wish to interface to DataFusion the recommendation is to create a Rust wrapper crate to interface @@ -197,7 +197,7 @@ and it is easy to implement on any struct that implements `Session`. [api docs]: http://docs.rs/datafusion-ffi/latest [rust abi]: https://doc.rust-lang.org/reference/abi.html [ffi]: https://doc.rust-lang.org/nomicon/ffi.html -[abi_stable]: https://crates.io/crates/abi_stable +[stabby]: https://crates.io/crates/stabby [async-ffi]: https://crates.io/crates/async-ffi [bindgen]: https://crates.io/crates/bindgen [`datafusion-python`]: https://datafusion.apache.org/python/ diff --git a/datafusion/ffi/src/arrow_wrappers.rs b/datafusion/ffi/src/arrow_wrappers.rs index c83e412310e7f..1c921b0f83b1e 100644 --- a/datafusion/ffi/src/arrow_wrappers.rs +++ b/datafusion/ffi/src/arrow_wrappers.rs @@ -17,7 +17,6 @@ use std::sync::Arc; -use abi_stable::StableAbi; use arrow::array::{ArrayRef, make_array}; use arrow::datatypes::{Schema, SchemaRef}; use arrow::error::ArrowError; @@ -26,10 +25,10 @@ use datafusion_common::{DataFusionError, ScalarValue}; use log::error; /// This is a wrapper struct around FFI_ArrowSchema simply to indicate -/// to the StableAbi macros that the underlying struct is FFI safe. +/// that the underlying struct is FFI safe. #[repr(C)] -#[derive(Debug, StableAbi)] -pub struct WrappedSchema(#[sabi(unsafe_opaque_field)] pub FFI_ArrowSchema); +#[derive(Debug)] +pub struct WrappedSchema(pub FFI_ArrowSchema); impl From for WrappedSchema { fn from(value: SchemaRef) -> Self { @@ -66,15 +65,13 @@ impl From for SchemaRef { } } -/// This is a wrapper struct for FFI_ArrowArray to indicate to StableAbi +/// This is a wrapper struct for FFI_ArrowArray to indicate /// that the struct is FFI Safe. For convenience, we also include the /// schema needed to create a record batch from the array. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct WrappedArray { - #[sabi(unsafe_opaque_field)] pub array: FFI_ArrowArray, - pub schema: WrappedSchema, } diff --git a/datafusion/ffi/src/catalog_provider.rs b/datafusion/ffi/src/catalog_provider.rs index ff588a89a71b3..922b982042b56 100644 --- a/datafusion/ffi/src/catalog_provider.rs +++ b/datafusion/ffi/src/catalog_provider.rs @@ -19,45 +19,45 @@ use std::any::Any; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RString, RVec}; use datafusion_catalog::{CatalogProvider, SchemaProvider}; use datafusion_common::error::Result; use datafusion_proto::logical_plan::{ DefaultLogicalExtensionCodec, LogicalExtensionCodec, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::execution::FFI_TaskContextProvider; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use crate::schema_provider::{FFI_SchemaProvider, ForeignSchemaProvider}; -use crate::util::FFIResult; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Option, FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing [`CatalogProvider`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_CatalogProvider { - pub schema_names: unsafe extern "C" fn(provider: &Self) -> RVec, + pub schema_names: unsafe extern "C" fn(provider: &Self) -> SVec, pub schema: unsafe extern "C" fn( provider: &Self, - name: RString, - ) -> ROption, - - pub register_schema: unsafe extern "C" fn( - provider: &Self, - name: RString, - schema: &FFI_SchemaProvider, - ) - -> FFIResult>, - - pub deregister_schema: unsafe extern "C" fn( - provider: &Self, - name: RString, - cascade: bool, - ) - -> FFIResult>, + name: SString, + ) -> FFI_Option, + + pub register_schema: + unsafe extern "C" fn( + provider: &Self, + name: SString, + schema: &FFI_SchemaProvider, + ) -> FFIResult>, + + pub deregister_schema: + unsafe extern "C" fn( + provider: &Self, + name: SString, + cascade: bool, + ) -> FFIResult>, pub logical_codec: FFI_LogicalExtensionCodec, @@ -107,7 +107,7 @@ impl FFI_CatalogProvider { unsafe extern "C" fn schema_names_fn_wrapper( provider: &FFI_CatalogProvider, -) -> RVec { +) -> SVec { unsafe { let names = provider.inner().schema_names(); names.into_iter().map(|s| s.into()).collect() @@ -116,8 +116,8 @@ unsafe extern "C" fn schema_names_fn_wrapper( unsafe extern "C" fn schema_fn_wrapper( provider: &FFI_CatalogProvider, - name: RString, -) -> ROption { + name: SString, +) -> FFI_Option { unsafe { let maybe_schema = provider.inner().schema(name.as_str()); maybe_schema @@ -134,16 +134,16 @@ unsafe extern "C" fn schema_fn_wrapper( unsafe extern "C" fn register_schema_fn_wrapper( provider: &FFI_CatalogProvider, - name: RString, + name: SString, schema: &FFI_SchemaProvider, -) -> FFIResult> { +) -> FFIResult> { unsafe { let runtime = provider.runtime(); let inner_provider = provider.inner(); let schema: Arc = schema.into(); let returned_schema = - rresult_return!(inner_provider.register_schema(name.as_str(), schema)) + sresult_return!(inner_provider.register_schema(name.as_str(), schema)) .map(|schema| { FFI_SchemaProvider::new_with_ffi_codec( schema, @@ -153,23 +153,23 @@ unsafe extern "C" fn register_schema_fn_wrapper( }) .into(); - RResult::ROk(returned_schema) + FFI_Result::Ok(returned_schema) } } unsafe extern "C" fn deregister_schema_fn_wrapper( provider: &FFI_CatalogProvider, - name: RString, + name: SString, cascade: bool, -) -> FFIResult> { +) -> FFIResult> { unsafe { let runtime = provider.runtime(); let inner_provider = provider.inner(); let maybe_schema = - rresult_return!(inner_provider.deregister_schema(name.as_str(), cascade)); + sresult_return!(inner_provider.deregister_schema(name.as_str(), cascade)); - RResult::ROk( + FFI_Result::Ok( maybe_schema .map(|schema| { FFI_SchemaProvider::new_with_ffi_codec( diff --git a/datafusion/ffi/src/catalog_provider_list.rs b/datafusion/ffi/src/catalog_provider_list.rs index 65574a7ac33de..3798dda8b846d 100644 --- a/datafusion/ffi/src/catalog_provider_list.rs +++ b/datafusion/ffi/src/catalog_provider_list.rs @@ -19,35 +19,36 @@ use std::any::Any; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RString, RVec}; use datafusion_catalog::{CatalogProvider, CatalogProviderList}; use datafusion_proto::logical_plan::{ DefaultLogicalExtensionCodec, LogicalExtensionCodec, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::catalog_provider::{FFI_CatalogProvider, ForeignCatalogProvider}; use crate::execution::FFI_TaskContextProvider; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; +use crate::util::FFI_Option; /// A stable struct for sharing [`CatalogProviderList`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_CatalogProviderList { /// Register a catalog pub register_catalog: unsafe extern "C" fn( &Self, - name: RString, + name: SString, catalog: &FFI_CatalogProvider, - ) -> ROption, + ) -> FFI_Option, /// List of existing catalogs - pub catalog_names: unsafe extern "C" fn(&Self) -> RVec, + pub catalog_names: unsafe extern "C" fn(&Self) -> SVec, /// Access a catalog pub catalog: - unsafe extern "C" fn(&Self, name: RString) -> ROption, + unsafe extern "C" fn(&Self, name: SString) -> FFI_Option, pub logical_codec: FFI_LogicalExtensionCodec, @@ -97,7 +98,7 @@ impl FFI_CatalogProviderList { unsafe extern "C" fn catalog_names_fn_wrapper( provider: &FFI_CatalogProviderList, -) -> RVec { +) -> SVec { unsafe { let names = provider.inner().catalog_names(); names.into_iter().map(|s| s.into()).collect() @@ -106,9 +107,9 @@ unsafe extern "C" fn catalog_names_fn_wrapper( unsafe extern "C" fn register_catalog_fn_wrapper( provider: &FFI_CatalogProviderList, - name: RString, + name: SString, catalog: &FFI_CatalogProvider, -) -> ROption { +) -> FFI_Option { unsafe { let runtime = provider.runtime(); let inner_provider = provider.inner(); @@ -129,8 +130,8 @@ unsafe extern "C" fn register_catalog_fn_wrapper( unsafe extern "C" fn catalog_fn_wrapper( provider: &FFI_CatalogProviderList, - name: RString, -) -> ROption { + name: SString, +) -> FFI_Option { unsafe { let runtime = provider.runtime(); let inner_provider = provider.inner(); diff --git a/datafusion/ffi/src/config/extension_options.rs b/datafusion/ffi/src/config/extension_options.rs index 48fd4e710921a..27d329cea5e1e 100644 --- a/datafusion/ffi/src/config/extension_options.rs +++ b/datafusion/ffi/src/config/extension_options.rs @@ -19,12 +19,15 @@ use std::any::Any; use std::collections::HashMap; use std::ffi::c_void; -use abi_stable::StableAbi; -use abi_stable::std_types::{RResult, RStr, RString, RVec, Tuple2}; use datafusion_common::config::{ConfigEntry, ConfigExtension, ExtensionOptions}; use datafusion_common::{Result, exec_err}; +use stabby::str::Str as SStr; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; + use crate::df_result; +use crate::util::{FFI_Result, FFIResult}; /// A stable struct for sharing [`ExtensionOptions`] across FFI boundaries. /// @@ -38,17 +41,16 @@ use crate::df_result; /// are stored with the full path prefix to avoid overwriting values when using /// multiple extensions. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_ExtensionOptions { /// Return a deep clone of this [`ExtensionOptions`] pub cloned: unsafe extern "C" fn(&Self) -> FFI_ExtensionOptions, /// Set the given `key`, `value` pair - pub set: - unsafe extern "C" fn(&mut Self, key: RStr, value: RStr) -> RResult<(), RString>, + pub set: unsafe extern "C" fn(&mut Self, key: SStr, value: SStr) -> FFIResult<()>, /// Returns the [`ConfigEntry`] stored in this [`ExtensionOptions`] - pub entries: unsafe extern "C" fn(&Self) -> RVec>, + pub entries: unsafe extern "C" fn(&Self) -> SVec<(SString, SString)>, /// Release the memory of the private data when it is no longer being used. pub release: unsafe extern "C" fn(&mut Self), @@ -91,20 +93,22 @@ unsafe extern "C" fn cloned_fn_wrapper( unsafe extern "C" fn set_fn_wrapper( options: &mut FFI_ExtensionOptions, - key: RStr, - value: RStr, -) -> RResult<(), RString> { - let _ = options.inner_mut().insert(key.into(), value.into()); - RResult::ROk(()) + key: SStr, + value: SStr, +) -> FFIResult<()> { + let _ = options + .inner_mut() + .insert(key.as_str().into(), value.as_str().into()); + FFI_Result::Ok(()) } unsafe extern "C" fn entries_fn_wrapper( options: &FFI_ExtensionOptions, -) -> RVec> { +) -> SVec<(SString, SString)> { options .inner() .iter() - .map(|(key, value)| (key.to_owned().into(), value.to_owned().into()).into()) + .map(|(key, value)| (key.to_owned().into(), value.to_owned().into())) .collect() } diff --git a/datafusion/ffi/src/config/mod.rs b/datafusion/ffi/src/config/mod.rs index 850a4dc337336..4c834b0ac03cc 100644 --- a/datafusion/ffi/src/config/mod.rs +++ b/datafusion/ffi/src/config/mod.rs @@ -17,12 +17,12 @@ pub mod extension_options; -use abi_stable::StableAbi; -use abi_stable::std_types::{RHashMap, RString}; use datafusion_common::config::{ ConfigExtension, ConfigOptions, ExtensionOptions, TableOptions, }; use datafusion_common::{DataFusionError, Result}; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use crate::config::extension_options::FFI_ExtensionOptions; @@ -32,16 +32,16 @@ use crate::config::extension_options::FFI_ExtensionOptions; /// than local extensions. The trait [`ExtensionOptionsFFIProvider`] can /// be used to simplify accessing FFI extensions. #[repr(C)] -#[derive(Debug, Clone, StableAbi)] +#[derive(Debug, Clone)] pub struct FFI_ConfigOptions { - base_options: RHashMap, + base_options: SVec<(SString, SString)>, extensions: FFI_ExtensionOptions, } impl From<&ConfigOptions> for FFI_ConfigOptions { fn from(options: &ConfigOptions) -> Self { - let base_options: RHashMap = options + let base_options: SVec<(SString, SString)> = options .entries() .into_iter() .filter_map(|entry| entry.value.map(|value| (entry.key, value))) @@ -120,16 +120,16 @@ impl ExtensionOptionsFFIProvider for TableOptions { /// than local extensions. The trait [`ExtensionOptionsFFIProvider`] can /// be used to simplify accessing FFI extensions. #[repr(C)] -#[derive(Debug, Clone, StableAbi)] +#[derive(Debug, Clone)] pub struct FFI_TableOptions { - base_options: RHashMap, + base_options: SVec<(SString, SString)>, extensions: FFI_ExtensionOptions, } impl From<&TableOptions> for FFI_TableOptions { fn from(options: &TableOptions) -> Self { - let base_options: RHashMap = options + let base_options: SVec<(SString, SString)> = options .entries() .into_iter() .filter_map(|entry| entry.value.map(|value| (entry.key, value))) diff --git a/datafusion/ffi/src/execution/task_ctx.rs b/datafusion/ffi/src/execution/task_ctx.rs index e0598db0a0170..d977e5602987d 100644 --- a/datafusion/ffi/src/execution/task_ctx.rs +++ b/datafusion/ffi/src/execution/task_ctx.rs @@ -18,9 +18,6 @@ use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::pmr::ROption; -use abi_stable::std_types::{RHashMap, RString}; use datafusion_execution::TaskContext; use datafusion_execution::config::SessionConfig; use datafusion_execution::runtime_env::RuntimeEnv; @@ -28,33 +25,37 @@ use datafusion_expr::{ AggregateUDF, AggregateUDFImpl, ScalarUDF, ScalarUDFImpl, WindowUDF, WindowUDFImpl, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; + use crate::session::config::FFI_SessionConfig; use crate::udaf::FFI_AggregateUDF; use crate::udf::FFI_ScalarUDF; use crate::udwf::FFI_WindowUDF; +use crate::util::FFI_Option; /// A stable struct for sharing [`TaskContext`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_TaskContext { /// Return the session ID. - pub session_id: unsafe extern "C" fn(&Self) -> RString, + pub session_id: unsafe extern "C" fn(&Self) -> SString, /// Return the task ID. - pub task_id: unsafe extern "C" fn(&Self) -> ROption, + pub task_id: unsafe extern "C" fn(&Self) -> FFI_Option, /// Return the session configuration. pub session_config: unsafe extern "C" fn(&Self) -> FFI_SessionConfig, - /// Returns a hashmap of names to scalar functions. - pub scalar_functions: unsafe extern "C" fn(&Self) -> RHashMap, + /// Returns a vec of name-function pairs for scalar functions. + pub scalar_functions: unsafe extern "C" fn(&Self) -> SVec<(SString, FFI_ScalarUDF)>, - /// Returns a hashmap of names to aggregate functions. + /// Returns a vec of name-function pairs for aggregate functions. pub aggregate_functions: - unsafe extern "C" fn(&Self) -> RHashMap, + unsafe extern "C" fn(&Self) -> SVec<(SString, FFI_AggregateUDF)>, - /// Returns a hashmap of names to window functions. - pub window_functions: unsafe extern "C" fn(&Self) -> RHashMap, + /// Returns a vec of name-function pairs for window functions. + pub window_functions: unsafe extern "C" fn(&Self) -> SVec<(SString, FFI_WindowUDF)>, /// Release the memory of the private data when it is no longer being used. pub release: unsafe extern "C" fn(arg: &mut Self), @@ -82,14 +83,14 @@ impl FFI_TaskContext { } } -unsafe extern "C" fn session_id_fn_wrapper(ctx: &FFI_TaskContext) -> RString { +unsafe extern "C" fn session_id_fn_wrapper(ctx: &FFI_TaskContext) -> SString { unsafe { let ctx = ctx.inner(); ctx.session_id().into() } } -unsafe extern "C" fn task_id_fn_wrapper(ctx: &FFI_TaskContext) -> ROption { +unsafe extern "C" fn task_id_fn_wrapper(ctx: &FFI_TaskContext) -> FFI_Option { unsafe { let ctx = ctx.inner(); ctx.task_id().map(|s| s.as_str().into()).into() @@ -107,7 +108,7 @@ unsafe extern "C" fn session_config_fn_wrapper( unsafe extern "C" fn scalar_functions_fn_wrapper( ctx: &FFI_TaskContext, -) -> RHashMap { +) -> SVec<(SString, FFI_ScalarUDF)> { unsafe { let ctx = ctx.inner(); ctx.scalar_functions() @@ -119,7 +120,7 @@ unsafe extern "C" fn scalar_functions_fn_wrapper( unsafe extern "C" fn aggregate_functions_fn_wrapper( ctx: &FFI_TaskContext, -) -> RHashMap { +) -> SVec<(SString, FFI_AggregateUDF)> { unsafe { let ctx = ctx.inner(); ctx.aggregate_functions() @@ -136,7 +137,7 @@ unsafe extern "C" fn aggregate_functions_fn_wrapper( unsafe extern "C" fn window_functions_fn_wrapper( ctx: &FFI_TaskContext, -) -> RHashMap { +) -> SVec<(SString, FFI_WindowUDF)> { unsafe { let ctx = ctx.inner(); ctx.window_functions() @@ -198,7 +199,7 @@ impl From for Arc { let udf = >::from(&kv_pair.1); ( - kv_pair.0.into_string(), + kv_pair.0.to_string(), Arc::new(ScalarUDF::new_from_shared_impl(udf)), ) }) @@ -209,7 +210,7 @@ impl From for Arc { let udaf = >::from(&kv_pair.1); ( - kv_pair.0.into_string(), + kv_pair.0.to_string(), Arc::new(AggregateUDF::new_from_shared_impl(udaf)), ) }) @@ -220,7 +221,7 @@ impl From for Arc { let udwf = >::from(&kv_pair.1); ( - kv_pair.0.into_string(), + kv_pair.0.to_string(), Arc::new(WindowUDF::new_from_shared_impl(udwf)), ) }) diff --git a/datafusion/ffi/src/execution/task_ctx_provider.rs b/datafusion/ffi/src/execution/task_ctx_provider.rs index 5d4eaac83975a..e2df3832e4b71 100644 --- a/datafusion/ffi/src/execution/task_ctx_provider.rs +++ b/datafusion/ffi/src/execution/task_ctx_provider.rs @@ -18,13 +18,12 @@ use std::ffi::c_void; use std::sync::{Arc, Weak}; -use abi_stable::StableAbi; use datafusion_common::{DataFusionError, ffi_datafusion_err}; use datafusion_execution::{TaskContext, TaskContextProvider}; use crate::execution::task_ctx::FFI_TaskContext; use crate::util::FFIResult; -use crate::{df_result, rresult}; +use crate::{df_result, sresult}; /// Struct for accessing the [`TaskContext`]. This method contains a weak /// reference, so there are no guarantees that the [`TaskContext`] remains @@ -32,7 +31,7 @@ use crate::{df_result, rresult}; /// data passed across the FFI boundary. See the crate README for /// additional information. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_TaskContextProvider { /// Retrieve the current [`TaskContext`] provided the provider has not /// gone out of scope. This function will return an error if the weakly @@ -77,7 +76,7 @@ unsafe extern "C" fn task_ctx_fn_wrapper( ctx_provider: &FFI_TaskContextProvider, ) -> FFIResult { unsafe { - rresult!( + sresult!( ctx_provider .inner() .map(FFI_TaskContext::from) diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rs index eba16d9390787..9c566b7432592 100644 --- a/datafusion/ffi/src/execution_plan.rs +++ b/datafusion/ffi/src/execution_plan.rs @@ -19,8 +19,6 @@ use std::ffi::c_void; use std::pin::Pin; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RString, RVec}; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{DataFusionError, Result}; @@ -28,30 +26,32 @@ use datafusion_execution::{SendableRecordBatchStream, TaskContext}; use datafusion_physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::config::FFI_ConfigOptions; use crate::execution::FFI_TaskContext; use crate::plan_properties::FFI_PlanProperties; use crate::record_batch_stream::FFI_RecordBatchStream; -use crate::util::FFIResult; -use crate::{df_result, rresult, rresult_return}; +use crate::util::{FFI_Option, FFIResult}; +use crate::{df_result, sresult, sresult_return}; /// A stable struct for sharing a [`ExecutionPlan`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_ExecutionPlan { /// Return the plan properties pub properties: unsafe extern "C" fn(plan: &Self) -> FFI_PlanProperties, /// Return a vector of children plans - pub children: unsafe extern "C" fn(plan: &Self) -> RVec, + pub children: unsafe extern "C" fn(plan: &Self) -> SVec, pub with_new_children: - unsafe extern "C" fn(plan: &Self, children: RVec) -> FFIResult, + unsafe extern "C" fn(plan: &Self, children: SVec) -> FFIResult, /// Return the plan name. - pub name: unsafe extern "C" fn(plan: &Self) -> RString, + pub name: unsafe extern "C" fn(plan: &Self) -> SString, /// Execute the plan and return a record batch stream. Errors /// will be returned as a string. @@ -65,7 +65,8 @@ pub struct FFI_ExecutionPlan { plan: &Self, target_partitions: usize, config: FFI_ConfigOptions, - ) -> FFIResult>, + ) + -> FFIResult>, /// Used to create a clone on the provider of the execution plan. This should /// only need to be called by the receiver of the plan. @@ -112,35 +113,31 @@ unsafe extern "C" fn properties_fn_wrapper( unsafe extern "C" fn children_fn_wrapper( plan: &FFI_ExecutionPlan, -) -> RVec { +) -> SVec { let runtime = plan.runtime(); - let plan = plan.inner(); - - let children: Vec<_> = plan + plan.inner() .children() .into_iter() .map(|child| FFI_ExecutionPlan::new(Arc::clone(child), runtime.clone())) - .collect(); - - children.into() + .collect() } unsafe extern "C" fn with_new_children_fn_wrapper( plan: &FFI_ExecutionPlan, - children: RVec, + children: SVec, ) -> FFIResult { let runtime = plan.runtime(); - let plan = Arc::clone(plan.inner()); - let children = rresult_return!( - children - .iter() - .map(>::try_from) - .collect::>>() - ); + let inner_plan = Arc::clone(plan.inner()); + + let children: Result>> = children + .iter() + .map(>::try_from) + .collect(); - let new_plan = rresult_return!(plan.with_new_children(children)); + let children = sresult_return!(children); + let new_plan = sresult_return!(inner_plan.with_new_children(children)); - RResult::ROk(FFI_ExecutionPlan::new(new_plan, runtime)) + crate::ffi_option::FFI_Result::Ok(FFI_ExecutionPlan::new(new_plan, runtime)) } unsafe extern "C" fn execute_fn_wrapper( @@ -154,7 +151,7 @@ unsafe extern "C" fn execute_fn_wrapper( let _runtime_guard = runtime.as_ref().map(|rt| rt.enter()); - rresult!( + sresult!( plan.execute(partition, ctx) .map(|rbs| FFI_RecordBatchStream::new(rbs, runtime)) ) @@ -164,13 +161,13 @@ unsafe extern "C" fn repartitioned_fn_wrapper( plan: &FFI_ExecutionPlan, target_partitions: usize, config: FFI_ConfigOptions, -) -> FFIResult> { +) -> FFIResult> { let maybe_config: Result = config.try_into(); - let config = rresult_return!(maybe_config); + let config = sresult_return!(maybe_config); let runtime = plan.runtime(); let plan = plan.inner(); - rresult!( + sresult!( plan.repartitioned(target_partitions, &config) .map(|maybe_plan| maybe_plan .map(|plan| FFI_ExecutionPlan::new(plan, runtime)) @@ -178,7 +175,7 @@ unsafe extern "C" fn repartitioned_fn_wrapper( ) } -unsafe extern "C" fn name_fn_wrapper(plan: &FFI_ExecutionPlan) -> RString { +unsafe extern "C" fn name_fn_wrapper(plan: &FFI_ExecutionPlan) -> SString { plan.inner().name().into() } @@ -388,7 +385,7 @@ impl ExecutionPlan for ForeignExecutionPlan { let children = children .into_iter() .map(|child| FFI_ExecutionPlan::new(child, None)) - .collect::>(); + .collect::>(); let new_plan = unsafe { df_result!((self.plan.with_new_children)(&self.plan, children))? }; diff --git a/datafusion/ffi/src/expr/columnar_value.rs b/datafusion/ffi/src/expr/columnar_value.rs index 7ad7645ecb6cf..19ad9ff7a3b79 100644 --- a/datafusion/ffi/src/expr/columnar_value.rs +++ b/datafusion/ffi/src/expr/columnar_value.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use datafusion_common::{DataFusionError, ScalarValue}; use datafusion_expr::ColumnarValue; @@ -23,8 +22,8 @@ use crate::arrow_wrappers::WrappedArray; /// A stable struct for sharing [`ColumnarValue`] across FFI boundaries. /// Scalar values are passed as an Arrow array of length 1. -#[repr(C)] -#[derive(Debug, StableAbi)] +#[repr(C, u8)] +#[derive(Debug)] pub enum FFI_ColumnarValue { Array(WrappedArray), Scalar(WrappedArray), diff --git a/datafusion/ffi/src/expr/distribution.rs b/datafusion/ffi/src/expr/distribution.rs index b9ebfc2362c7a..ca760f16ad17c 100644 --- a/datafusion/ffi/src/expr/distribution.rs +++ b/datafusion/ffi/src/expr/distribution.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use datafusion_common::DataFusionError; use datafusion_expr::statistics::{ BernoulliDistribution, Distribution, ExponentialDistribution, GaussianDistribution, @@ -28,7 +27,7 @@ use crate::expr::interval::FFI_Interval; /// A stable struct for sharing [`Distribution`] across FFI boundaries. /// See ['Distribution'] for the meaning of each variant. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] #[expect(clippy::large_enum_variant)] pub enum FFI_Distribution { Uniform(FFI_UniformDistribution), @@ -67,13 +66,13 @@ impl TryFrom for Distribution { } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_UniformDistribution { interval: FFI_Interval, } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_ExponentialDistribution { rate: WrappedArray, offset: WrappedArray, @@ -81,20 +80,20 @@ pub struct FFI_ExponentialDistribution { } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_GaussianDistribution { mean: WrappedArray, variance: WrappedArray, } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_BernoulliDistribution { p: WrappedArray, } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_GenericDistribution { mean: WrappedArray, median: WrappedArray, diff --git a/datafusion/ffi/src/expr/expr_properties.rs b/datafusion/ffi/src/expr/expr_properties.rs index 199a399a6471f..5b37cc6a28535 100644 --- a/datafusion/ffi/src/expr/expr_properties.rs +++ b/datafusion/ffi/src/expr/expr_properties.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use arrow_schema::SortOptions; use datafusion_common::DataFusionError; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; @@ -25,7 +24,7 @@ use crate::expr::interval::FFI_Interval; /// A stable struct for sharing [`ExprProperties`] across FFI boundaries. /// See [`ExprProperties`] for the meaning of each field. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_ExprProperties { sort_properties: FFI_SortProperties, range: FFI_Interval, @@ -60,7 +59,7 @@ impl TryFrom for ExprProperties { } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub enum FFI_SortProperties { Ordered(FFI_SortOptions), Unordered, @@ -88,7 +87,7 @@ impl From<&FFI_SortProperties> for SortProperties { } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_SortOptions { pub descending: bool, pub nulls_first: bool, diff --git a/datafusion/ffi/src/expr/interval.rs b/datafusion/ffi/src/expr/interval.rs index 450f3747a57f0..6334f7bb24d90 100644 --- a/datafusion/ffi/src/expr/interval.rs +++ b/datafusion/ffi/src/expr/interval.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use datafusion_common::DataFusionError; use datafusion_expr::interval_arithmetic::Interval; @@ -25,7 +24,7 @@ use crate::arrow_wrappers::WrappedArray; /// See [`Interval`] for the meaning of each field. Scalar values /// are passed as Arrow arrays of length 1. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_Interval { lower: WrappedArray, upper: WrappedArray, diff --git a/datafusion/ffi/src/ffi_option.rs b/datafusion/ffi/src/ffi_option.rs new file mode 100644 index 0000000000000..d2bfbb1e38c96 --- /dev/null +++ b/datafusion/ffi/src/ffi_option.rs @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! FFI-safe Option and Result types that do not require `IStable` bounds. +//! +//! stabby's `Option` and `Result` require `T: IStable` for niche +//! optimization. Many of our FFI structs contain self-referential function +//! pointers and cannot implement `IStable`. These simple `#[repr(C)]` types +//! provide the same FFI-safe semantics without that constraint. + +/// An FFI-safe option type. +#[repr(C, u8)] +#[derive(Debug, Clone)] +pub enum FFI_Option { + Some(T), + None, +} + +impl From> for FFI_Option { + fn from(opt: Option) -> Self { + match opt { + Some(v) => FFI_Option::Some(v), + None => FFI_Option::None, + } + } +} + +impl From> for Option { + fn from(opt: FFI_Option) -> Self { + match opt { + FFI_Option::Some(v) => Some(v), + FFI_Option::None => None, + } + } +} + +impl FFI_Option { + pub fn as_ref(&self) -> Option<&T> { + match self { + FFI_Option::Some(v) => Some(v), + FFI_Option::None => None, + } + } + + pub fn map U>(self, f: F) -> FFI_Option { + match self { + FFI_Option::Some(v) => FFI_Option::Some(f(v)), + FFI_Option::None => FFI_Option::None, + } + } + + pub fn into_option(self) -> Option { + self.into() + } +} + +/// An FFI-safe result type. +#[repr(C, u8)] +#[derive(Debug, Clone)] +pub enum FFI_Result { + Ok(T), + Err(E), +} + +impl From> for FFI_Result { + fn from(res: Result) -> Self { + match res { + Ok(v) => FFI_Result::Ok(v), + Err(e) => FFI_Result::Err(e), + } + } +} + +impl From> for Result { + fn from(res: FFI_Result) -> Self { + match res { + FFI_Result::Ok(v) => Ok(v), + FFI_Result::Err(e) => Err(e), + } + } +} + +impl FFI_Result { + pub fn is_ok(&self) -> bool { + matches!(self, FFI_Result::Ok(_)) + } + + pub fn is_err(&self) -> bool { + matches!(self, FFI_Result::Err(_)) + } + + pub fn unwrap_err(self) -> E { + match self { + FFI_Result::Err(e) => e, + FFI_Result::Ok(_) => panic!("called unwrap_err on Ok"), + } + } + + pub fn map U>(self, f: F) -> FFI_Result { + match self { + FFI_Result::Ok(v) => FFI_Result::Ok(f(v)), + FFI_Result::Err(e) => FFI_Result::Err(e), + } + } + + pub fn into_result(self) -> Result { + self.into() + } +} + +impl PartialEq for FFI_Result { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (FFI_Result::Ok(a), FFI_Result::Ok(b)) => a == b, + (FFI_Result::Err(a), FFI_Result::Err(b)) => a == b, + _ => false, + } + } +} diff --git a/datafusion/ffi/src/insert_op.rs b/datafusion/ffi/src/insert_op.rs index 6471039105e80..e264d24dcaee2 100644 --- a/datafusion/ffi/src/insert_op.rs +++ b/datafusion/ffi/src/insert_op.rs @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use datafusion_expr::logical_plan::dml::InsertOp; /// FFI safe version of [`InsertOp`]. -#[repr(C)] -#[derive(StableAbi)] +#[expect(non_camel_case_types)] +#[repr(u8)] pub enum FFI_InsertOp { Append, Overwrite, diff --git a/datafusion/ffi/src/lib.rs b/datafusion/ffi/src/lib.rs index d7410e8483735..279a6a2cdc0fd 100644 --- a/datafusion/ffi/src/lib.rs +++ b/datafusion/ffi/src/lib.rs @@ -32,6 +32,7 @@ pub mod config; pub mod execution; pub mod execution_plan; pub mod expr; +pub mod ffi_option; pub mod insert_op; pub mod physical_expr; pub mod plan_properties; @@ -51,6 +52,7 @@ pub mod volatility; #[cfg(feature = "integration-tests")] pub mod tests; +mod physical_optimizer; /// Returns the major version of the FFI implementation. If the API evolves, /// we use the major version to identify compatibility over the unsafe diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rs index 189a1e478217e..2459eafa2ce08 100644 --- a/datafusion/ffi/src/physical_expr/mod.rs +++ b/datafusion/ffi/src/physical_expr/mod.rs @@ -24,8 +24,6 @@ use std::fmt::{Display, Formatter}; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RString, RVec}; use arrow::array::{ArrayRef, BooleanArray, RecordBatch}; use arrow::datatypes::SchemaRef; use arrow_schema::ffi::FFI_ArrowSchema; @@ -38,6 +36,9 @@ use datafusion_expr::statistics::Distribution; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr_common::physical_expr::fmt_sql; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; + use crate::arrow_wrappers::{WrappedArray, WrappedSchema}; use crate::expr::columnar_value::FFI_ColumnarValue; use crate::expr::distribution::FFI_Distribution; @@ -46,11 +47,11 @@ use crate::expr::interval::FFI_Interval; use crate::record_batch_stream::{ record_batch_to_wrapped_array, wrapped_array_to_record_batch, }; -use crate::util::FFIResult; -use crate::{df_result, rresult, rresult_return}; +use crate::util::{FFI_Option, FFI_Result, FFIResult}; +use crate::{df_result, sresult, sresult_return}; #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_PhysicalExpr { pub data_type: unsafe extern "C" fn( &Self, @@ -74,50 +75,50 @@ pub struct FFI_PhysicalExpr { selection: WrappedArray, ) -> FFIResult, - pub children: unsafe extern "C" fn(&Self) -> RVec, + pub children: unsafe extern "C" fn(&Self) -> SVec, pub new_with_children: - unsafe extern "C" fn(&Self, children: &RVec) -> FFIResult, + unsafe extern "C" fn(&Self, children: &SVec) -> FFIResult, pub evaluate_bounds: unsafe extern "C" fn( &Self, - children: RVec, + children: SVec, ) -> FFIResult, pub propagate_constraints: unsafe extern "C" fn( &Self, interval: FFI_Interval, - children: RVec, - ) -> FFIResult>>, + children: SVec, + ) -> FFIResult>>, pub evaluate_statistics: unsafe extern "C" fn( &Self, - children: RVec, + children: SVec, ) -> FFIResult, pub propagate_statistics: unsafe extern "C" fn( &Self, parent: FFI_Distribution, - children: RVec, - ) -> FFIResult>>, + children: SVec, + ) -> FFIResult>>, pub get_properties: unsafe extern "C" fn( &Self, - children: RVec, + children: SVec, ) -> FFIResult, - pub fmt_sql: unsafe extern "C" fn(&Self) -> FFIResult, + pub fmt_sql: unsafe extern "C" fn(&Self) -> FFIResult, - pub snapshot: unsafe extern "C" fn(&Self) -> FFIResult>, + pub snapshot: unsafe extern "C" fn(&Self) -> FFIResult>, pub snapshot_generation: unsafe extern "C" fn(&Self) -> u64, pub is_volatile_node: unsafe extern "C" fn(&Self) -> bool, // Display trait - pub display: unsafe extern "C" fn(&Self) -> RString, + pub display: unsafe extern "C" fn(&Self) -> SString, // Hash trait pub hash: unsafe extern "C" fn(&Self) -> u64, @@ -167,7 +168,7 @@ unsafe extern "C" fn data_type_fn_wrapper( .data_type(&schema) .and_then(|dt| FFI_ArrowSchema::try_from(dt).map_err(Into::into)) .map(WrappedSchema); - rresult!(data_type) + sresult!(data_type) } unsafe extern "C" fn nullable_fn_wrapper( @@ -176,15 +177,15 @@ unsafe extern "C" fn nullable_fn_wrapper( ) -> FFIResult { let expr = expr.inner(); let schema: SchemaRef = input_schema.into(); - rresult!(expr.nullable(&schema)) + sresult!(expr.nullable(&schema)) } unsafe extern "C" fn evaluate_fn_wrapper( expr: &FFI_PhysicalExpr, batch: WrappedArray, ) -> FFIResult { - let batch = rresult_return!(wrapped_array_to_record_batch(batch)); - rresult!( + let batch = sresult_return!(wrapped_array_to_record_batch(batch)); + sresult!( expr.inner() .evaluate(&batch) .and_then(FFI_ColumnarValue::try_from) @@ -197,7 +198,7 @@ unsafe extern "C" fn return_field_fn_wrapper( ) -> FFIResult { let expr = expr.inner(); let schema: SchemaRef = input_schema.into(); - rresult!( + sresult!( expr.return_field(&schema) .and_then(|f| FFI_ArrowSchema::try_from(&f).map_err(Into::into)) .map(WrappedSchema) @@ -209,15 +210,15 @@ unsafe extern "C" fn evaluate_selection_fn_wrapper( batch: WrappedArray, selection: WrappedArray, ) -> FFIResult { - let batch = rresult_return!(wrapped_array_to_record_batch(batch)); - let selection: ArrayRef = rresult_return!(selection.try_into()); - let selection = rresult_return!( + let batch = sresult_return!(wrapped_array_to_record_batch(batch)); + let selection: ArrayRef = sresult_return!(selection.try_into()); + let selection = sresult_return!( selection .as_any() .downcast_ref::() .ok_or(ffi_datafusion_err!("Unexpected selection array type")) ); - rresult!( + sresult!( expr.inner() .evaluate_selection(&batch, selection) .and_then(FFI_ColumnarValue::try_from) @@ -226,7 +227,7 @@ unsafe extern "C" fn evaluate_selection_fn_wrapper( unsafe extern "C" fn children_fn_wrapper( expr: &FFI_PhysicalExpr, -) -> RVec { +) -> SVec { let expr = expr.inner(); let children = expr.children(); children @@ -237,19 +238,19 @@ unsafe extern "C" fn children_fn_wrapper( unsafe extern "C" fn new_with_children_fn_wrapper( expr: &FFI_PhysicalExpr, - children: &RVec, + children: &SVec, ) -> FFIResult { let expr = Arc::clone(expr.inner()); let children = children.iter().map(Into::into).collect::>(); - rresult!(expr.with_new_children(children).map(FFI_PhysicalExpr::from)) + sresult!(expr.with_new_children(children).map(FFI_PhysicalExpr::from)) } unsafe extern "C" fn evaluate_bounds_fn_wrapper( expr: &FFI_PhysicalExpr, - children: RVec, + children: SVec, ) -> FFIResult { let expr = expr.inner(); - let children = rresult_return!( + let children = sresult_return!( children .into_iter() .map(Interval::try_from) @@ -257,7 +258,7 @@ unsafe extern "C" fn evaluate_bounds_fn_wrapper( ); let children_borrowed = children.iter().collect::>(); - rresult!( + sresult!( expr.evaluate_bounds(&children_borrowed) .and_then(FFI_Interval::try_from) ) @@ -266,11 +267,11 @@ unsafe extern "C" fn evaluate_bounds_fn_wrapper( unsafe extern "C" fn propagate_constraints_fn_wrapper( expr: &FFI_PhysicalExpr, interval: FFI_Interval, - children: RVec, -) -> FFIResult>> { + children: SVec, +) -> FFIResult>> { let expr = expr.inner(); - let interval = rresult_return!(Interval::try_from(interval)); - let children = rresult_return!( + let interval = sresult_return!(Interval::try_from(interval)); + let children = sresult_return!( children .into_iter() .map(Interval::try_from) @@ -279,33 +280,33 @@ unsafe extern "C" fn propagate_constraints_fn_wrapper( let children_borrowed = children.iter().collect::>(); let result = - rresult_return!(expr.propagate_constraints(&interval, &children_borrowed)); + sresult_return!(expr.propagate_constraints(&interval, &children_borrowed)); - let result = rresult_return!( + let result = sresult_return!( result .map(|intervals| intervals .into_iter() .map(FFI_Interval::try_from) - .collect::>>()) + .collect::>>()) .transpose() ); - RResult::ROk(result.into()) + FFI_Result::Ok(result.into()) } unsafe extern "C" fn evaluate_statistics_fn_wrapper( expr: &FFI_PhysicalExpr, - children: RVec, + children: SVec, ) -> FFIResult { let expr = expr.inner(); - let children = rresult_return!( + let children = sresult_return!( children .into_iter() .map(Distribution::try_from) .collect::>>() ); let children_borrowed = children.iter().collect::>(); - rresult!( + sresult!( expr.evaluate_statistics(&children_borrowed) .and_then(|dist| FFI_Distribution::try_from(&dist)) ) @@ -314,11 +315,11 @@ unsafe extern "C" fn evaluate_statistics_fn_wrapper( unsafe extern "C" fn propagate_statistics_fn_wrapper( expr: &FFI_PhysicalExpr, parent: FFI_Distribution, - children: RVec, -) -> FFIResult>> { + children: SVec, +) -> FFIResult>> { let expr = expr.inner(); - let parent = rresult_return!(Distribution::try_from(parent)); - let children = rresult_return!( + let parent = sresult_return!(Distribution::try_from(parent)); + let children = sresult_return!( children .into_iter() .map(Distribution::try_from) @@ -326,47 +327,47 @@ unsafe extern "C" fn propagate_statistics_fn_wrapper( ); let children_borrowed = children.iter().collect::>(); - let result = rresult_return!(expr.propagate_statistics(&parent, &children_borrowed)); - let result = rresult_return!( + let result = sresult_return!(expr.propagate_statistics(&parent, &children_borrowed)); + let result = sresult_return!( result .map(|dists| dists .iter() .map(FFI_Distribution::try_from) - .collect::>>()) + .collect::>>()) .transpose() ); - RResult::ROk(result.into()) + FFI_Result::Ok(result.into()) } unsafe extern "C" fn get_properties_fn_wrapper( expr: &FFI_PhysicalExpr, - children: RVec, + children: SVec, ) -> FFIResult { let expr = expr.inner(); - let children = rresult_return!( + let children = sresult_return!( children .into_iter() .map(ExprProperties::try_from) .collect::>>() ); - rresult!( + sresult!( expr.get_properties(&children) .and_then(|p| FFI_ExprProperties::try_from(&p)) ) } -unsafe extern "C" fn fmt_sql_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFIResult { +unsafe extern "C" fn fmt_sql_fn_wrapper(expr: &FFI_PhysicalExpr) -> FFIResult { let expr = expr.inner(); let result = fmt_sql(expr.as_ref()).to_string(); - RResult::ROk(result.into()) + FFI_Result::Ok(result.into()) } unsafe extern "C" fn snapshot_fn_wrapper( expr: &FFI_PhysicalExpr, -) -> FFIResult> { +) -> FFIResult> { let expr = expr.inner(); - rresult!( + sresult!( expr.snapshot() .map(|snapshot| snapshot.map(FFI_PhysicalExpr::from).into()) ) @@ -381,7 +382,7 @@ unsafe extern "C" fn is_volatile_node_fn_wrapper(expr: &FFI_PhysicalExpr) -> boo let expr = expr.inner(); expr.is_volatile_node() } -unsafe extern "C" fn display_fn_wrapper(expr: &FFI_PhysicalExpr) -> RString { +unsafe extern "C" fn display_fn_wrapper(expr: &FFI_PhysicalExpr) -> SString { let expr = expr.inner(); format!("{expr}").into() } @@ -599,7 +600,7 @@ impl PhysicalExpr for ForeignPhysicalExpr { let children = children .iter() .map(|interval| FFI_Interval::try_from(*interval)) - .collect::>>()?; + .collect::>>()?; df_result!((self.expr.evaluate_bounds)(&self.expr, children)) .and_then(Interval::try_from) } @@ -615,7 +616,7 @@ impl PhysicalExpr for ForeignPhysicalExpr { let children = children .iter() .map(|interval| FFI_Interval::try_from(*interval)) - .collect::>>()?; + .collect::>>()?; let result = df_result!((self.expr.propagate_constraints)( &self.expr, interval, children ))?; @@ -637,7 +638,7 @@ impl PhysicalExpr for ForeignPhysicalExpr { let children = children .iter() .map(|dist| FFI_Distribution::try_from(*dist)) - .collect::>>()?; + .collect::>>()?; let result = df_result!((self.expr.evaluate_statistics)(&self.expr, children))?; @@ -655,7 +656,7 @@ impl PhysicalExpr for ForeignPhysicalExpr { let children = children .iter() .map(|dist| FFI_Distribution::try_from(*dist)) - .collect::>>()?; + .collect::>>()?; let result = df_result!((self.expr.propagate_statistics)( &self.expr, parent, children ))?; @@ -678,7 +679,7 @@ impl PhysicalExpr for ForeignPhysicalExpr { let children = children .iter() .map(FFI_ExprProperties::try_from) - .collect::>>()?; + .collect::>>()?; df_result!((self.expr.get_properties)(&self.expr, children)) .and_then(ExprProperties::try_from) } @@ -687,8 +688,8 @@ impl PhysicalExpr for ForeignPhysicalExpr { fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { unsafe { match (self.expr.fmt_sql)(&self.expr) { - RResult::ROk(sql) => write!(f, "{sql}"), - RResult::RErr(_) => Err(std::fmt::Error), + FFI_Result::Ok(sql) => write!(f, "{sql}"), + FFI_Result::Err(_) => Err(std::fmt::Error), } } } diff --git a/datafusion/ffi/src/physical_expr/partitioning.rs b/datafusion/ffi/src/physical_expr/partitioning.rs index cda4fd2c97f45..434b6a097e645 100644 --- a/datafusion/ffi/src/physical_expr/partitioning.rs +++ b/datafusion/ffi/src/physical_expr/partitioning.rs @@ -17,20 +17,19 @@ use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::RVec; use datafusion_physical_expr::Partitioning; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; +use stabby::vec::Vec as SVec; use crate::physical_expr::FFI_PhysicalExpr; /// A stable struct for sharing [`Partitioning`] across FFI boundaries. /// See ['Partitioning'] for the meaning of each variant. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub enum FFI_Partitioning { RoundRobinBatch(usize), - Hash(RVec, usize), + Hash(SVec, usize), UnknownPartitioning(usize), } diff --git a/datafusion/ffi/src/physical_expr/sort.rs b/datafusion/ffi/src/physical_expr/sort.rs index fd3339b10555a..fc8e2a81f36eb 100644 --- a/datafusion/ffi/src/physical_expr/sort.rs +++ b/datafusion/ffi/src/physical_expr/sort.rs @@ -17,7 +17,6 @@ use std::sync::Arc; -use abi_stable::StableAbi; use arrow_schema::SortOptions; use datafusion_physical_expr::PhysicalSortExpr; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; @@ -28,7 +27,7 @@ use crate::physical_expr::FFI_PhysicalExpr; /// A stable struct for sharing [`PhysicalSortExpr`] across FFI boundaries. /// See [`PhysicalSortExpr`] for the meaning of each field. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_PhysicalSortExpr { expr: FFI_PhysicalExpr, options: FFI_SortOptions, diff --git a/datafusion/ffi/src/physical_optimizer.rs b/datafusion/ffi/src/physical_optimizer.rs new file mode 100644 index 0000000000000..7fbb426321f16 --- /dev/null +++ b/datafusion/ffi/src/physical_optimizer.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::ffi::c_void; +use std::sync::Arc; +use stabby::stabby; +use stabby::string::String as SString; +use datafusion::physical_optimizer::PhysicalOptimizerRule; +use crate::config::FFI_ConfigOptions; +use crate::execution_plan::FFI_ExecutionPlan; +use crate::util::FFIResult; + +#[repr(C)] +pub struct FFI_PhysicalOptimizerRule { + + /// FFI equivalent to the `name` of a [`PhysicalOptimizerRule`] + pub name: unsafe extern "C" fn(rule : &Self) -> SString, + pub schema_check: unsafe extern "C" fn(rule : &Self) -> bool, + pub optimize: unsafe extern "C" fn ( + rule: &Self, + plan: FFI_ExecutionPlan, + config: FFI_ConfigOptions, + ) -> FFIResult, + + pub clone: unsafe extern "C" fn(rule: &Self) -> Self, + pub release: unsafe extern "C" fn(rule: &mut Self), + pub private_data: *mut c_void, + +} + +unsafe impl Send for FFI_PhysicalOptimizerRule {} +unsafe impl Sync for FFI_PhysicalOptimizerRule {} + +struct PhysicalOptimizerRulePrivateData { + rule: Arc, +} diff --git a/datafusion/ffi/src/plan_properties.rs b/datafusion/ffi/src/plan_properties.rs index d009de3f04b99..b286ee2d7d30c 100644 --- a/datafusion/ffi/src/plan_properties.rs +++ b/datafusion/ffi/src/plan_properties.rs @@ -18,8 +18,6 @@ use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RVec}; use arrow::datatypes::SchemaRef; use datafusion_common::error::{DataFusionError, Result}; use datafusion_physical_expr::EquivalenceProperties; @@ -27,13 +25,16 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; use datafusion_physical_plan::PlanProperties; use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; +use stabby::vec::Vec as SVec; + use crate::arrow_wrappers::WrappedSchema; use crate::physical_expr::partitioning::FFI_Partitioning; use crate::physical_expr::sort::FFI_PhysicalSortExpr; +use crate::util::FFI_Option; /// A stable struct for sharing [`PlanProperties`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_PlanProperties { /// The output partitioning of the plan. pub output_partitioning: unsafe extern "C" fn(plan: &Self) -> FFI_Partitioning, @@ -46,7 +47,7 @@ pub struct FFI_PlanProperties { /// The output ordering of the plan. pub output_ordering: - unsafe extern "C" fn(plan: &Self) -> ROption>, + unsafe extern "C" fn(plan: &Self) -> FFI_Option>, /// Return the schema of the plan. pub schema: unsafe extern "C" fn(plan: &Self) -> WrappedSchema, @@ -95,8 +96,8 @@ unsafe extern "C" fn boundedness_fn_wrapper( unsafe extern "C" fn output_ordering_fn_wrapper( properties: &FFI_PlanProperties, -) -> ROption> { - let ordering: Option> = +) -> FFI_Option> { + let ordering: Option> = properties.inner().output_ordering().map(|lex_ordering| { let vec_ordering: Vec = lex_ordering.clone().into(); vec_ordering @@ -159,7 +160,7 @@ impl TryFrom for PlanProperties { let ffi_schema = unsafe { (ffi_props.schema)(&ffi_props) }; let schema = (&ffi_schema.0).try_into()?; - let ffi_orderings: Option> = + let ffi_orderings: Option> = unsafe { (ffi_props.output_ordering)(&ffi_props) }.into(); let sort_exprs = ffi_orderings .map(|ordering_vec| { @@ -194,8 +195,8 @@ impl TryFrom for PlanProperties { } /// FFI safe version of [`Boundedness`]. -#[repr(C)] -#[derive(Clone, StableAbi)] +#[repr(C, u8)] +#[derive(Clone)] pub enum FFI_Boundedness { Bounded, Unbounded { requires_infinite_memory: bool }, @@ -228,8 +229,9 @@ impl From for Boundedness { } /// FFI safe version of [`EmissionType`]. -#[repr(C)] -#[derive(Clone, StableAbi)] +#[expect(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone)] pub enum FFI_EmissionType { Incremental, Final, diff --git a/datafusion/ffi/src/proto/logical_extension_codec.rs b/datafusion/ffi/src/proto/logical_extension_codec.rs index 2beeead7039c0..20b2613faa9b7 100644 --- a/datafusion/ffi/src/proto/logical_extension_codec.rs +++ b/datafusion/ffi/src/proto/logical_extension_codec.rs @@ -19,8 +19,6 @@ use std::any::Any; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{RResult, RSlice, RStr, RVec}; use arrow::datatypes::SchemaRef; use datafusion_catalog::TableProvider; use datafusion_common::error::Result; @@ -34,6 +32,10 @@ use datafusion_expr::{ use datafusion_proto::logical_plan::{ DefaultLogicalExtensionCodec, LogicalExtensionCodec, }; + +use stabby::slice::Slice as SSlice; +use stabby::str::Str as SStr; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::arrow_wrappers::WrappedSchema; @@ -42,60 +44,60 @@ use crate::table_provider::FFI_TableProvider; use crate::udaf::FFI_AggregateUDF; use crate::udf::FFI_ScalarUDF; use crate::udwf::FFI_WindowUDF; -use crate::util::FFIResult; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing [`LogicalExtensionCodec`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_LogicalExtensionCodec { /// Decode bytes into a table provider. try_decode_table_provider: unsafe extern "C" fn( &Self, - buf: RSlice, - table_ref: RStr, + buf: SSlice, + table_ref: SStr, schema: WrappedSchema, ) -> FFIResult, /// Encode a table provider into bytes. try_encode_table_provider: unsafe extern "C" fn( &Self, - table_ref: RStr, + table_ref: SStr, node: FFI_TableProvider, - ) -> FFIResult>, + ) -> FFIResult>, /// Decode bytes into a user defined scalar function. try_decode_udf: unsafe extern "C" fn( &Self, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult, /// Encode a user defined scalar function into bytes. try_encode_udf: - unsafe extern "C" fn(&Self, node: FFI_ScalarUDF) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_ScalarUDF) -> FFIResult>, /// Decode bytes into a user defined aggregate function. try_decode_udaf: unsafe extern "C" fn( &Self, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult, /// Encode a user defined aggregate function into bytes. try_encode_udaf: - unsafe extern "C" fn(&Self, node: FFI_AggregateUDF) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_AggregateUDF) -> FFIResult>, /// Decode bytes into a user defined window function. try_decode_udwf: unsafe extern "C" fn( &Self, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult, /// Encode a user defined window function into bytes. try_encode_udwf: - unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFIResult>, pub task_ctx_provider: FFI_TaskContextProvider, @@ -144,24 +146,24 @@ impl FFI_LogicalExtensionCodec { unsafe extern "C" fn try_decode_table_provider_fn_wrapper( codec: &FFI_LogicalExtensionCodec, - buf: RSlice, - table_ref: RStr, + buf: SSlice, + table_ref: SStr, schema: WrappedSchema, ) -> FFIResult { - let ctx = rresult_return!(codec.task_ctx()); + let ctx = sresult_return!(codec.task_ctx()); let runtime = codec.runtime().clone(); let codec_inner = codec.inner(); let table_ref = TableReference::from(table_ref.as_str()); let schema: SchemaRef = schema.into(); - let table_provider = rresult_return!(codec_inner.try_decode_table_provider( + let table_provider = sresult_return!(codec_inner.try_decode_table_provider( buf.as_ref(), &table_ref, schema, ctx.as_ref() )); - RResult::ROk(FFI_TableProvider::new_with_ffi_codec( + FFI_Result::Ok(FFI_TableProvider::new_with_ffi_codec( table_provider, true, runtime, @@ -171,100 +173,100 @@ unsafe extern "C" fn try_decode_table_provider_fn_wrapper( unsafe extern "C" fn try_encode_table_provider_fn_wrapper( codec: &FFI_LogicalExtensionCodec, - table_ref: RStr, + table_ref: SStr, node: FFI_TableProvider, -) -> FFIResult> { +) -> FFIResult> { let table_ref = TableReference::from(table_ref.as_str()); let table_provider: Arc = (&node).into(); let codec = codec.inner(); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_table_provider( + sresult_return!(codec.try_encode_table_provider( &table_ref, table_provider, &mut bytes )); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn try_decode_udf_fn_wrapper( codec: &FFI_LogicalExtensionCodec, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult { let codec = codec.inner(); - let udf = rresult_return!(codec.try_decode_udf(name.as_str(), buf.as_ref())); + let udf = sresult_return!(codec.try_decode_udf(name.as_str(), buf.as_ref())); let udf = FFI_ScalarUDF::from(udf); - RResult::ROk(udf) + FFI_Result::Ok(udf) } unsafe extern "C" fn try_encode_udf_fn_wrapper( codec: &FFI_LogicalExtensionCodec, node: FFI_ScalarUDF, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); let node: Arc = (&node).into(); let node = ScalarUDF::new_from_shared_impl(node); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_udf(&node, &mut bytes)); + sresult_return!(codec.try_encode_udf(&node, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn try_decode_udaf_fn_wrapper( codec: &FFI_LogicalExtensionCodec, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult { let codec_inner = codec.inner(); - let udaf = rresult_return!(codec_inner.try_decode_udaf(name.into(), buf.as_ref())); + let udaf = sresult_return!(codec_inner.try_decode_udaf(name.into(), buf.as_ref())); let udaf = FFI_AggregateUDF::from(udaf); - RResult::ROk(udaf) + FFI_Result::Ok(udaf) } unsafe extern "C" fn try_encode_udaf_fn_wrapper( codec: &FFI_LogicalExtensionCodec, node: FFI_AggregateUDF, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); let udaf: Arc = (&node).into(); let udaf = AggregateUDF::new_from_shared_impl(udaf); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_udaf(&udaf, &mut bytes)); + sresult_return!(codec.try_encode_udaf(&udaf, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn try_decode_udwf_fn_wrapper( codec: &FFI_LogicalExtensionCodec, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult { let codec = codec.inner(); - let udwf = rresult_return!(codec.try_decode_udwf(name.into(), buf.as_ref())); + let udwf = sresult_return!(codec.try_decode_udwf(name.into(), buf.as_ref())); let udwf = FFI_WindowUDF::from(udwf); - RResult::ROk(udwf) + FFI_Result::Ok(udwf) } unsafe extern "C" fn try_encode_udwf_fn_wrapper( codec: &FFI_LogicalExtensionCodec, node: FFI_WindowUDF, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); let udwf: Arc = (&node).into(); let udwf = WindowUDF::new_from_shared_impl(udwf); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_udwf(&udwf, &mut bytes)); + sresult_return!(codec.try_encode_udwf(&udwf, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_LogicalExtensionCodec) { diff --git a/datafusion/ffi/src/proto/physical_extension_codec.rs b/datafusion/ffi/src/proto/physical_extension_codec.rs index 9d5c2e8af3a6c..8c5b8ba1f8a13 100644 --- a/datafusion/ffi/src/proto/physical_extension_codec.rs +++ b/datafusion/ffi/src/proto/physical_extension_codec.rs @@ -15,8 +15,10 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; -use abi_stable::std_types::{RResult, RSlice, RStr, RVec}; +use std::any::Any; +use std::ffi::c_void; +use std::sync::Arc; + use datafusion_common::error::Result; use datafusion_execution::TaskContext; use datafusion_expr::{ @@ -24,7 +26,10 @@ use datafusion_expr::{ }; use datafusion_physical_plan::ExecutionPlan; use datafusion_proto::physical_plan::PhysicalExtensionCodec; -use std::{any::Any, ffi::c_void, sync::Arc}; + +use stabby::slice::Slice as SSlice; +use stabby::str::Str as SStr; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::execution::FFI_TaskContextProvider; @@ -32,56 +37,56 @@ use crate::execution_plan::FFI_ExecutionPlan; use crate::udaf::FFI_AggregateUDF; use crate::udf::FFI_ScalarUDF; use crate::udwf::FFI_WindowUDF; -use crate::util::FFIResult; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing [`PhysicalExtensionCodec`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_PhysicalExtensionCodec { /// Decode bytes into an execution plan. try_decode: unsafe extern "C" fn( &Self, - buf: RSlice, - inputs: RVec, + buf: SSlice, + inputs: SVec, ) -> FFIResult, /// Encode an execution plan into bytes. try_encode: - unsafe extern "C" fn(&Self, node: FFI_ExecutionPlan) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_ExecutionPlan) -> FFIResult>, /// Decode bytes into a user defined scalar function. try_decode_udf: unsafe extern "C" fn( &Self, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult, /// Encode a user defined scalar function into bytes. try_encode_udf: - unsafe extern "C" fn(&Self, node: FFI_ScalarUDF) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_ScalarUDF) -> FFIResult>, /// Decode bytes into a user defined aggregate function. try_decode_udaf: unsafe extern "C" fn( &Self, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult, /// Encode a user defined aggregate function into bytes. try_encode_udaf: - unsafe extern "C" fn(&Self, node: FFI_AggregateUDF) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_AggregateUDF) -> FFIResult>, /// Decode bytes into a user defined window function. try_decode_udwf: unsafe extern "C" fn( &Self, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult, /// Encode a user defined window function into bytes. try_encode_udwf: - unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFIResult>, + unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFIResult>, /// Access the current [`TaskContext`]. task_ctx_provider: FFI_TaskContextProvider, @@ -127,116 +132,116 @@ impl FFI_PhysicalExtensionCodec { unsafe extern "C" fn try_decode_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, - buf: RSlice, - inputs: RVec, + buf: SSlice, + inputs: SVec, ) -> FFIResult { let runtime = codec.runtime().clone(); let task_ctx: Arc = - rresult_return!((&codec.task_ctx_provider).try_into()); + sresult_return!((&codec.task_ctx_provider).try_into()); let codec = codec.inner(); let inputs = inputs .into_iter() .map(|plan| >::try_from(&plan)) .collect::>>(); - let inputs = rresult_return!(inputs); + let inputs = sresult_return!(inputs); let plan = - rresult_return!(codec.try_decode(buf.as_ref(), &inputs, task_ctx.as_ref())); + sresult_return!(codec.try_decode(buf.as_ref(), &inputs, task_ctx.as_ref())); - RResult::ROk(FFI_ExecutionPlan::new(plan, runtime)) + FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime)) } unsafe extern "C" fn try_encode_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, node: FFI_ExecutionPlan, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); - let plan: Arc = rresult_return!((&node).try_into()); + let plan: Arc = sresult_return!((&node).try_into()); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode(plan, &mut bytes)); + sresult_return!(codec.try_encode(plan, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn try_decode_udf_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult { let codec = codec.inner(); - let udf = rresult_return!(codec.try_decode_udf(name.as_str(), buf.as_ref())); + let udf = sresult_return!(codec.try_decode_udf(name.as_str(), buf.as_ref())); let udf = FFI_ScalarUDF::from(udf); - RResult::ROk(udf) + FFI_Result::Ok(udf) } unsafe extern "C" fn try_encode_udf_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, node: FFI_ScalarUDF, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); let node: Arc = (&node).into(); let node = ScalarUDF::new_from_shared_impl(node); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_udf(&node, &mut bytes)); + sresult_return!(codec.try_encode_udf(&node, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn try_decode_udaf_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult { let codec_inner = codec.inner(); - let udaf = rresult_return!(codec_inner.try_decode_udaf(name.into(), buf.as_ref())); + let udaf = sresult_return!(codec_inner.try_decode_udaf(name.into(), buf.as_ref())); let udaf = FFI_AggregateUDF::from(udaf); - RResult::ROk(udaf) + FFI_Result::Ok(udaf) } unsafe extern "C" fn try_encode_udaf_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, node: FFI_AggregateUDF, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); let udaf: Arc = (&node).into(); let udaf = AggregateUDF::new_from_shared_impl(udaf); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_udaf(&udaf, &mut bytes)); + sresult_return!(codec.try_encode_udaf(&udaf, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn try_decode_udwf_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, - name: RStr, - buf: RSlice, + name: SStr, + buf: SSlice, ) -> FFIResult { let codec = codec.inner(); - let udwf = rresult_return!(codec.try_decode_udwf(name.into(), buf.as_ref())); + let udwf = sresult_return!(codec.try_decode_udwf(name.into(), buf.as_ref())); let udwf = FFI_WindowUDF::from(udwf); - RResult::ROk(udwf) + FFI_Result::Ok(udwf) } unsafe extern "C" fn try_encode_udwf_fn_wrapper( codec: &FFI_PhysicalExtensionCodec, node: FFI_WindowUDF, -) -> FFIResult> { +) -> FFIResult> { let codec = codec.inner(); let udwf: Arc = (&node).into(); let udwf = WindowUDF::new_from_shared_impl(udwf); let mut bytes = Vec::new(); - rresult_return!(codec.try_encode_udwf(&udwf, &mut bytes)); + sresult_return!(codec.try_encode_udwf(&udwf, &mut bytes)); - RResult::ROk(bytes.into()) + FFI_Result::Ok(bytes.into_iter().collect()) } unsafe extern "C" fn release_fn_wrapper(codec: &mut FFI_PhysicalExtensionCodec) { diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rs index 53078a0e4bbae..8e0fefc0078fe 100644 --- a/datafusion/ffi/src/record_batch_stream.rs +++ b/datafusion/ffi/src/record_batch_stream.rs @@ -18,31 +18,31 @@ use std::ffi::c_void; use std::task::Poll; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult}; use arrow::array::{Array, RecordBatch, StructArray, make_array}; use arrow::ffi::{from_ffi, to_ffi}; use async_ffi::{ContextExt, FfiContext, FfiPoll}; use datafusion_common::{DataFusionError, Result, ffi_datafusion_err, ffi_err}; use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream}; use futures::{Stream, TryStreamExt}; + use tokio::runtime::Handle; use crate::arrow_wrappers::{WrappedArray, WrappedSchema}; -use crate::rresult; -use crate::util::FFIResult; +use crate::sresult; +use crate::util::{FFI_Option, FFI_Result, FFIResult}; /// A stable struct for sharing [`RecordBatchStream`] across FFI boundaries. /// We use the async-ffi crate for handling async calls across libraries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_RecordBatchStream { /// This mirrors the `poll_next` of [`RecordBatchStream`] but does so /// in a FFI safe manner. pub poll_next: unsafe extern "C" fn( stream: &Self, cx: &mut FfiContext, - ) -> FfiPoll>>, + ) + -> FfiPoll>>, /// Return the schema of the record batch pub schema: unsafe extern "C" fn(stream: &Self) -> WrappedSchema, @@ -107,7 +107,7 @@ pub(crate) fn record_batch_to_wrapped_array( ) -> FFIResult { let schema = WrappedSchema::from(record_batch.schema()); let struct_array = StructArray::from(record_batch); - rresult!( + sresult!( to_ffi(&struct_array.to_data()) .map(|(array, _schema)| WrappedArray { array, schema }) ) @@ -116,20 +116,20 @@ pub(crate) fn record_batch_to_wrapped_array( // probably want to use pub unsafe fn from_ffi(array: FFI_ArrowArray, schema: &FFI_ArrowSchema) -> Result { fn maybe_record_batch_to_wrapped_stream( record_batch: Option>, -) -> ROption> { +) -> FFI_Option> { match record_batch { Some(Ok(record_batch)) => { - ROption::RSome(record_batch_to_wrapped_array(record_batch)) + FFI_Option::Some(record_batch_to_wrapped_array(record_batch)) } - Some(Err(e)) => ROption::RSome(RResult::RErr(e.to_string().into())), - None => ROption::RNone, + Some(Err(e)) => FFI_Option::Some(FFI_Result::Err(e.to_string().into())), + None => FFI_Option::None, } } unsafe extern "C" fn poll_next_fn_wrapper( stream: &FFI_RecordBatchStream, cx: &mut FfiContext, -) -> FfiPoll>> { +) -> FfiPoll>> { unsafe { let private_data = stream.private_data as *mut RecordBatchStreamPrivateData; let stream = &mut (*private_data).rbs; @@ -171,14 +171,18 @@ pub(crate) fn wrapped_array_to_record_batch(array: WrappedArray) -> Result>, + array: FFI_Option>, ) -> Option> { + let array: Option> = array.into(); match array { - ROption::RSome(RResult::ROk(wrapped_array)) => { - Some(wrapped_array_to_record_batch(wrapped_array)) + Some(result) => { + let result: std::result::Result = result.into(); + match result { + Ok(wrapped_array) => Some(wrapped_array_to_record_batch(wrapped_array)), + Err(e) => Some(ffi_err!("{e}")), + } } - ROption::RSome(RResult::RErr(e)) => Some(ffi_err!("{e}")), - ROption::RNone => None, + None => None, } } diff --git a/datafusion/ffi/src/schema_provider.rs b/datafusion/ffi/src/schema_provider.rs index 5d1348e2328f7..f788c56614ba5 100644 --- a/datafusion/ffi/src/schema_provider.rs +++ b/datafusion/ffi/src/schema_provider.rs @@ -19,8 +19,6 @@ use std::any::Any; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RString, RVec}; use async_ffi::{FfiFuture, FutureExt}; use async_trait::async_trait; use datafusion_catalog::{SchemaProvider, TableProvider}; @@ -28,42 +26,45 @@ use datafusion_common::error::{DataFusionError, Result}; use datafusion_proto::logical_plan::{ DefaultLogicalExtensionCodec, LogicalExtensionCodec, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::execution::FFI_TaskContextProvider; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use crate::table_provider::{FFI_TableProvider, ForeignTableProvider}; -use crate::util::FFIResult; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Option, FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing [`SchemaProvider`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_SchemaProvider { - pub owner_name: ROption, + pub owner_name: FFI_Option, - pub table_names: unsafe extern "C" fn(provider: &Self) -> RVec, + pub table_names: unsafe extern "C" fn(provider: &Self) -> SVec, pub table: unsafe extern "C" fn( provider: &Self, - name: RString, - ) - -> FfiFuture>>, + name: SString, + ) -> FfiFuture< + FFIResult>, + >, pub register_table: unsafe extern "C" fn( provider: &Self, - name: RString, + name: SString, table: FFI_TableProvider, ) - -> FFIResult>, + -> FFIResult>, - pub deregister_table: unsafe extern "C" fn( - provider: &Self, - name: RString, - ) - -> FFIResult>, + pub deregister_table: + unsafe extern "C" fn( + provider: &Self, + name: SString, + ) -> FFIResult>, - pub table_exist: unsafe extern "C" fn(provider: &Self, name: RString) -> bool, + pub table_exist: unsafe extern "C" fn(provider: &Self, name: SString) -> bool, pub logical_codec: FFI_LogicalExtensionCodec, @@ -113,7 +114,7 @@ impl FFI_SchemaProvider { unsafe extern "C" fn table_names_fn_wrapper( provider: &FFI_SchemaProvider, -) -> RVec { +) -> SVec { unsafe { let provider = provider.inner(); @@ -124,21 +125,21 @@ unsafe extern "C" fn table_names_fn_wrapper( unsafe extern "C" fn table_fn_wrapper( provider: &FFI_SchemaProvider, - name: RString, -) -> FfiFuture>> { + name: SString, +) -> FfiFuture>> { unsafe { let runtime = provider.runtime(); let logical_codec = provider.logical_codec.clone(); let provider = Arc::clone(provider.inner()); async move { - let table = rresult_return!(provider.table(name.as_str()).await) + let table = sresult_return!(provider.table(name.as_str()).await) .map(|t| { FFI_TableProvider::new_with_ffi_codec(t, true, runtime, logical_codec) }) .into(); - RResult::ROk(table) + FFI_Result::Ok(table) } .into_ffi() } @@ -146,9 +147,9 @@ unsafe extern "C" fn table_fn_wrapper( unsafe extern "C" fn register_table_fn_wrapper( provider: &FFI_SchemaProvider, - name: RString, + name: SString, table: FFI_TableProvider, -) -> FFIResult> { +) -> FFIResult> { unsafe { let runtime = provider.runtime(); let logical_codec = provider.logical_codec.clone(); @@ -156,36 +157,36 @@ unsafe extern "C" fn register_table_fn_wrapper( let table = Arc::new(ForeignTableProvider(table)); - let returned_table = rresult_return!(provider.register_table(name.into(), table)) + let returned_table = sresult_return!(provider.register_table(name.into(), table)) .map(|t| { FFI_TableProvider::new_with_ffi_codec(t, true, runtime, logical_codec) }); - RResult::ROk(returned_table.into()) + FFI_Result::Ok(returned_table.into()) } } unsafe extern "C" fn deregister_table_fn_wrapper( provider: &FFI_SchemaProvider, - name: RString, -) -> FFIResult> { + name: SString, +) -> FFIResult> { unsafe { let runtime = provider.runtime(); let logical_codec = provider.logical_codec.clone(); let provider = provider.inner(); - let returned_table = rresult_return!(provider.deregister_table(name.as_str())) + let returned_table = sresult_return!(provider.deregister_table(name.as_str())) .map(|t| { FFI_TableProvider::new_with_ffi_codec(t, true, runtime, logical_codec) }); - RResult::ROk(returned_table.into()) + FFI_Result::Ok(returned_table.into()) } } unsafe extern "C" fn table_exist_fn_wrapper( provider: &FFI_SchemaProvider, - name: RString, + name: SString, ) -> bool { unsafe { provider.inner().table_exist(name.as_str()) } } @@ -318,7 +319,7 @@ impl SchemaProvider for ForeignSchemaProvider { } fn owner_name(&self) -> Option<&str> { - let name: Option<&RString> = self.0.owner_name.as_ref().into(); + let name: Option<&SString> = self.0.owner_name.as_ref(); name.map(|s| s.as_str()) } diff --git a/datafusion/ffi/src/session/config.rs b/datafusion/ffi/src/session/config.rs index 63f0f20ecc7d5..fca0190c07138 100644 --- a/datafusion/ffi/src/session/config.rs +++ b/datafusion/ffi/src/session/config.rs @@ -18,7 +18,6 @@ use std::ffi::c_void; use crate::config::FFI_ConfigOptions; -use abi_stable::StableAbi; use datafusion_common::config::ConfigOptions; use datafusion_common::error::{DataFusionError, Result}; use datafusion_execution::config::SessionConfig; @@ -35,7 +34,7 @@ use datafusion_execution::config::SessionConfig; /// SessionConfig via a FFI interface would be extensive and provide limited /// value over this version. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_SessionConfig { /// FFI stable configuration options. pub config_options: FFI_ConfigOptions, diff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rs index a72e043a77461..6b397a21b2d1b 100644 --- a/datafusion/ffi/src/session/mod.rs +++ b/datafusion/ffi/src/session/mod.rs @@ -20,8 +20,6 @@ use std::collections::HashMap; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{RHashMap, RResult, RStr, RString, RVec}; use arrow_schema::SchemaRef; use arrow_schema::ffi::FFI_ArrowSchema; use async_ffi::{FfiFuture, FutureExt}; @@ -45,6 +43,10 @@ use datafusion_proto::logical_plan::to_proto::serialize_expr; use datafusion_proto::protobuf::LogicalExprNode; use datafusion_session::Session; use prost::Message; + +use stabby::str::Str as SStr; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::arrow_wrappers::WrappedSchema; @@ -56,8 +58,8 @@ use crate::session::config::FFI_SessionConfig; use crate::udaf::FFI_AggregateUDF; use crate::udf::FFI_ScalarUDF; use crate::udwf::FFI_WindowUDF; -use crate::util::FFIResult; -use crate::{df_result, rresult, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult, sresult_return}; pub mod config; @@ -74,34 +76,33 @@ pub mod config; /// which has methods that require `&dyn Session`. For usage within this crate /// we know the [`Session`] lifetimes are valid. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub(crate) struct FFI_SessionRef { - session_id: unsafe extern "C" fn(&Self) -> RStr, + session_id: unsafe extern "C" fn(&Self) -> SStr, config: unsafe extern "C" fn(&Self) -> FFI_SessionConfig, create_physical_plan: unsafe extern "C" fn( &Self, - logical_plan_serialized: RVec, + logical_plan_serialized: SVec, ) -> FfiFuture>, create_physical_expr: unsafe extern "C" fn( &Self, - expr_serialized: RVec, + expr_serialized: SVec, schema: WrappedSchema, ) -> FFIResult, - scalar_functions: unsafe extern "C" fn(&Self) -> RHashMap, + scalar_functions: unsafe extern "C" fn(&Self) -> SVec<(SString, FFI_ScalarUDF)>, - aggregate_functions: - unsafe extern "C" fn(&Self) -> RHashMap, + aggregate_functions: unsafe extern "C" fn(&Self) -> SVec<(SString, FFI_AggregateUDF)>, - window_functions: unsafe extern "C" fn(&Self) -> RHashMap, + window_functions: unsafe extern "C" fn(&Self) -> SVec<(SString, FFI_WindowUDF)>, - table_options: unsafe extern "C" fn(&Self) -> RHashMap, + table_options: unsafe extern "C" fn(&Self) -> SVec<(SString, SString)>, - default_table_options: unsafe extern "C" fn(&Self) -> RHashMap, + default_table_options: unsafe extern "C" fn(&Self) -> SVec<(SString, SString)>, task_ctx: unsafe extern "C" fn(&Self) -> FFI_TaskContext, @@ -148,7 +149,7 @@ impl FFI_SessionRef { } } -unsafe extern "C" fn session_id_fn_wrapper(session: &FFI_SessionRef) -> RStr<'_> { +unsafe extern "C" fn session_id_fn_wrapper(session: &FFI_SessionRef) -> SStr<'_> { let session = session.inner(); session.session_id().into() } @@ -160,7 +161,7 @@ unsafe extern "C" fn config_fn_wrapper(session: &FFI_SessionRef) -> FFI_SessionC unsafe extern "C" fn create_physical_plan_fn_wrapper( session: &FFI_SessionRef, - logical_plan_serialized: RVec, + logical_plan_serialized: SVec, ) -> FfiFuture> { unsafe { let runtime = session.runtime().clone(); @@ -169,14 +170,14 @@ unsafe extern "C" fn create_physical_plan_fn_wrapper( let session = session.inner(); let task_ctx = session.task_ctx(); - let logical_plan = rresult_return!(logical_plan_from_bytes( + let logical_plan = sresult_return!(logical_plan_from_bytes( logical_plan_serialized.as_slice(), task_ctx.as_ref(), )); let physical_plan = session.create_physical_plan(&logical_plan).await; - rresult!(physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, runtime))) + sresult!(physical_plan.map(|plan| FFI_ExecutionPlan::new(plan, runtime))) } .into_ffi() } @@ -184,7 +185,7 @@ unsafe extern "C" fn create_physical_plan_fn_wrapper( unsafe extern "C" fn create_physical_expr_fn_wrapper( session: &FFI_SessionRef, - expr_serialized: RVec, + expr_serialized: SVec, schema: WrappedSchema, ) -> FFIResult { let codec: Arc = (&session.logical_codec).into(); @@ -194,17 +195,17 @@ unsafe extern "C" fn create_physical_expr_fn_wrapper( let logical_expr = parse_expr(&logical_expr, session.task_ctx().as_ref(), codec.as_ref()).unwrap(); let schema: SchemaRef = schema.into(); - let schema: DFSchema = rresult_return!(schema.try_into()); + let schema: DFSchema = sresult_return!(schema.try_into()); let physical_expr = - rresult_return!(session.create_physical_expr(logical_expr, &schema)); + sresult_return!(session.create_physical_expr(logical_expr, &schema)); - RResult::ROk(physical_expr.into()) + FFI_Result::Ok(physical_expr.into()) } unsafe extern "C" fn scalar_functions_fn_wrapper( session: &FFI_SessionRef, -) -> RHashMap { +) -> SVec<(SString, FFI_ScalarUDF)> { let session = session.inner(); session .scalar_functions() @@ -215,7 +216,7 @@ unsafe extern "C" fn scalar_functions_fn_wrapper( unsafe extern "C" fn aggregate_functions_fn_wrapper( session: &FFI_SessionRef, -) -> RHashMap { +) -> SVec<(SString, FFI_AggregateUDF)> { let session = session.inner(); session .aggregate_functions() @@ -231,7 +232,7 @@ unsafe extern "C" fn aggregate_functions_fn_wrapper( unsafe extern "C" fn window_functions_fn_wrapper( session: &FFI_SessionRef, -) -> RHashMap { +) -> SVec<(SString, FFI_WindowUDF)> { let session = session.inner(); session .window_functions() @@ -240,13 +241,13 @@ unsafe extern "C" fn window_functions_fn_wrapper( .collect() } -fn table_options_to_rhash(mut options: TableOptions) -> RHashMap { +fn table_options_to_rhash(mut options: TableOptions) -> SVec<(SString, SString)> { // It is important that we mutate options here and set current format // to None so that when we call `entries()` we get ALL format entries. // We will pass current_format as a special case and strip it on the // other side of the boundary. let current_format = options.current_format.take(); - let mut options: HashMap = options + let mut options: HashMap = options .entries() .into_iter() .filter_map(|entry| entry.value.map(|v| (entry.key.into(), v.into()))) @@ -263,12 +264,12 @@ fn table_options_to_rhash(mut options: TableOptions) -> RHashMap RHashMap { +) -> SVec<(SString, SString)> { let session = session.inner(); let table_options = session.table_options(); table_options_to_rhash(table_options.clone()) @@ -276,7 +277,7 @@ unsafe extern "C" fn table_options_fn_wrapper( unsafe extern "C" fn default_table_options_fn_wrapper( session: &FFI_SessionRef, -) -> RHashMap { +) -> SVec<(SString, SString)> { let session = session.inner(); let table_options = session.default_table_options(); @@ -411,7 +412,7 @@ impl TryFrom<&FFI_SessionRef> for ForeignSession { let udf = >::from(&kv_pair.1); ( - kv_pair.0.into_string(), + kv_pair.0.to_string(), Arc::new(ScalarUDF::new_from_shared_impl(udf)), ) }) @@ -422,7 +423,7 @@ impl TryFrom<&FFI_SessionRef> for ForeignSession { let udaf = >::from(&kv_pair.1); ( - kv_pair.0.into_string(), + kv_pair.0.to_string(), Arc::new(AggregateUDF::new_from_shared_impl(udaf)), ) }) @@ -433,7 +434,7 @@ impl TryFrom<&FFI_SessionRef> for ForeignSession { let udwf = >::from(&kv_pair.1); ( - kv_pair.0.into_string(), + kv_pair.0.to_string(), Arc::new(WindowUDF::new_from_shared_impl(udwf)), ) }) @@ -459,10 +460,10 @@ impl Clone for FFI_SessionRef { } } -fn table_options_from_rhashmap(options: RHashMap) -> TableOptions { +fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOptions { let mut options: HashMap = options .into_iter() - .map(|kv_pair| (kv_pair.0.into_string(), kv_pair.1.into_string())) + .map(|kv_pair| (kv_pair.0.to_string(), kv_pair.1.to_string())) .collect(); let current_format = options.remove("datafusion_ffi.table_current_format"); @@ -572,7 +573,7 @@ impl Session for ForeignSession { let physical_expr = df_result!((self.session.create_physical_expr)( &self.session, - logical_expr.into(), + logical_expr.into_iter().collect(), schema ))?; diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rs index 4a89bb025a56d..9b293bd449cd4 100644 --- a/datafusion/ffi/src/table_provider.rs +++ b/datafusion/ffi/src/table_provider.rs @@ -19,8 +19,6 @@ use std::any::Any; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RVec}; use arrow::datatypes::SchemaRef; use async_ffi::{FfiFuture, FutureExt}; use async_trait::async_trait; @@ -37,6 +35,8 @@ use datafusion_proto::logical_plan::{ }; use datafusion_proto::protobuf::LogicalExprList; use prost::Message; + +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use super::execution_plan::FFI_ExecutionPlan; @@ -46,8 +46,8 @@ use crate::execution::FFI_TaskContextProvider; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use crate::session::{FFI_SessionRef, ForeignSession}; use crate::table_source::{FFI_TableProviderFilterPushDown, FFI_TableType}; -use crate::util::FFIResult; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Option, FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing [`TableProvider`] across FFI boundaries. /// @@ -89,7 +89,7 @@ use crate::{df_result, rresult_return}; /// It is important to be careful when expanding these functions to be certain which /// side of the interface each object refers to. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_TableProvider { /// Return the table schema schema: unsafe extern "C" fn(provider: &Self) -> WrappedSchema, @@ -108,9 +108,9 @@ pub struct FFI_TableProvider { scan: unsafe extern "C" fn( provider: &Self, session: FFI_SessionRef, - projections: ROption>, - filters_serialized: RVec, - limit: ROption, + projections: FFI_Option>, + filters_serialized: SVec, + limit: FFI_Option, ) -> FfiFuture>, /// Return the type of table. See [`TableType`] for options. @@ -122,8 +122,8 @@ pub struct FFI_TableProvider { supports_filters_pushdown: Option< unsafe extern "C" fn( provider: &FFI_TableProvider, - filters_serialized: RVec, - ) -> FFIResult>, + filters_serialized: SVec, + ) -> FFIResult>, >, insert_into: unsafe extern "C" fn( @@ -190,7 +190,7 @@ fn supports_filters_pushdown_internal( filters_serialized: &[u8], task_ctx: &Arc, codec: &dyn LogicalExtensionCodec, -) -> Result> { +) -> Result> { let filters = match filters_serialized.is_empty() { true => vec![], false => { @@ -202,7 +202,7 @@ fn supports_filters_pushdown_internal( }; let filters_borrowed: Vec<&Expr> = filters.iter().collect(); - let results: RVec<_> = provider + let results: SVec<_> = provider .supports_filters_pushdown(&filters_borrowed)? .iter() .map(|v| v.into()) @@ -213,10 +213,10 @@ fn supports_filters_pushdown_internal( unsafe extern "C" fn supports_filters_pushdown_fn_wrapper( provider: &FFI_TableProvider, - filters_serialized: RVec, -) -> FFIResult> { + filters_serialized: SVec, +) -> FFIResult> { let logical_codec: Arc = (&provider.logical_codec).into(); - let task_ctx = rresult_return!(>::try_from( + let task_ctx = sresult_return!(>::try_from( &provider.logical_codec.task_ctx_provider )); supports_filters_pushdown_internal( @@ -232,9 +232,9 @@ unsafe extern "C" fn supports_filters_pushdown_fn_wrapper( unsafe extern "C" fn scan_fn_wrapper( provider: &FFI_TableProvider, session: FFI_SessionRef, - projections: ROption>, - filters_serialized: RVec, - limit: ROption, + projections: FFI_Option>, + filters_serialized: SVec, + limit: FFI_Option, ) -> FfiFuture> { let task_ctx: Result, DataFusionError> = (&provider.logical_codec.task_ctx_provider).try_into(); @@ -244,7 +244,7 @@ unsafe extern "C" fn scan_fn_wrapper( async move { let mut foreign_session = None; - let session = rresult_return!( + let session = sresult_return!( session .as_local() .map(Ok::<&(dyn Session + Send + Sync), DataFusionError>) @@ -254,14 +254,14 @@ unsafe extern "C" fn scan_fn_wrapper( }) ); - let task_ctx = rresult_return!(task_ctx); + let task_ctx = sresult_return!(task_ctx); let filters = match filters_serialized.is_empty() { true => vec![], false => { let proto_filters = - rresult_return!(LogicalExprList::decode(filters_serialized.as_ref())); + sresult_return!(LogicalExprList::decode(filters_serialized.as_ref())); - rresult_return!(parse_exprs( + sresult_return!(parse_exprs( proto_filters.expr.iter(), task_ctx.as_ref(), logical_codec.as_ref(), @@ -272,13 +272,13 @@ unsafe extern "C" fn scan_fn_wrapper( let projections: Option> = projections.into_option().map(|p| p.into_iter().collect()); - let plan = rresult_return!( + let plan = sresult_return!( internal_provider .scan(session, projections.as_ref(), &filters, limit.into()) .await ); - RResult::ROk(FFI_ExecutionPlan::new(plan, runtime.clone())) + FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime.clone())) } .into_ffi() } @@ -295,7 +295,7 @@ unsafe extern "C" fn insert_into_fn_wrapper( async move { let mut foreign_session = None; - let session = rresult_return!( + let session = sresult_return!( session .as_local() .map(Ok::<&(dyn Session + Send + Sync), DataFusionError>) @@ -305,17 +305,17 @@ unsafe extern "C" fn insert_into_fn_wrapper( }) ); - let input = rresult_return!(>::try_from(&input)); + let input = sresult_return!(>::try_from(&input)); let insert_op = InsertOp::from(insert_op); - let plan = rresult_return!( + let plan = sresult_return!( internal_provider .insert_into(session, input, insert_op) .await ); - RResult::ROk(FFI_ExecutionPlan::new(plan, runtime.clone())) + FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime.clone())) } .into_ffi() } @@ -465,7 +465,7 @@ impl TableProvider for ForeignTableProvider { ) -> Result> { let session = FFI_SessionRef::new(session, None, self.0.logical_codec.clone()); - let projections: ROption> = projection + let projections: FFI_Option> = projection .map(|p| p.iter().map(|v| v.to_owned()).collect()) .into(); @@ -473,7 +473,7 @@ impl TableProvider for ForeignTableProvider { let filter_list = LogicalExprList { expr: serialize_exprs(filters, codec.as_ref())?, }; - let filters_serialized = filter_list.encode_to_vec().into(); + let filters_serialized = filter_list.encode_to_vec().into_iter().collect(); let plan = unsafe { let maybe_plan = (self.0.scan)( @@ -518,7 +518,10 @@ impl TableProvider for ForeignTableProvider { }; let serialized_filters = expr_list.encode_to_vec(); - let pushdowns = df_result!(pushdown_fn(&self.0, serialized_filters.into()))?; + let pushdowns = df_result!(pushdown_fn( + &self.0, + serialized_filters.into_iter().collect() + ))?; Ok(pushdowns.iter().map(|v| v.into()).collect()) } diff --git a/datafusion/ffi/src/table_provider_factory.rs b/datafusion/ffi/src/table_provider_factory.rs index 15789eeab0421..c9bca3abbd512 100644 --- a/datafusion/ffi/src/table_provider_factory.rs +++ b/datafusion/ffi/src/table_provider_factory.rs @@ -17,10 +17,6 @@ use std::{ffi::c_void, sync::Arc}; -use abi_stable::{ - StableAbi, - std_types::{RResult, RString, RVec}, -}; use async_ffi::{FfiFuture, FutureExt}; use async_trait::async_trait; use datafusion_catalog::{Session, TableProvider, TableProviderFactory}; @@ -32,13 +28,16 @@ use datafusion_proto::logical_plan::{ }; use datafusion_proto::protobuf::LogicalPlanNode; use prost::Message; + +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::execution::FFI_TaskContextProvider; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use crate::session::{FFI_SessionRef, ForeignSession}; use crate::table_provider::{FFI_TableProvider, ForeignTableProvider}; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing [`TableProviderFactory`] across FFI boundaries. /// @@ -49,7 +48,7 @@ use crate::{df_result, rresult_return}; /// /// [`FFI_TableProvider`]: crate::table_provider::FFI_TableProvider #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_TableProviderFactory { /// Create a TableProvider with the given command. /// @@ -62,8 +61,8 @@ pub struct FFI_TableProviderFactory { create: unsafe extern "C" fn( factory: &Self, session: FFI_SessionRef, - cmd_serialized: RVec, - ) -> FfiFuture>, + cmd_serialized: SVec, + ) -> FfiFuture>, logical_codec: FFI_LogicalExtensionCodec, @@ -144,7 +143,7 @@ impl FFI_TableProviderFactory { fn deserialize_cmd( &self, - cmd_serialized: &RVec, + cmd_serialized: &SVec, ) -> Result { let task_ctx: Arc = (&self.logical_codec.task_ctx_provider).try_into()?; @@ -186,15 +185,15 @@ impl From<&FFI_TableProviderFactory> for Arc { unsafe extern "C" fn create_fn_wrapper( factory: &FFI_TableProviderFactory, session: FFI_SessionRef, - cmd_serialized: RVec, -) -> FfiFuture> { + cmd_serialized: SVec, +) -> FfiFuture> { let factory = factory.clone(); async move { - let provider = rresult_return!( + let provider = sresult_return!( create_fn_wrapper_impl(factory, session, cmd_serialized).await ); - RResult::ROk(provider) + FFI_Result::Ok(provider) } .into_ffi() } @@ -202,7 +201,7 @@ unsafe extern "C" fn create_fn_wrapper( async fn create_fn_wrapper_impl( factory: FFI_TableProviderFactory, session: FFI_SessionRef, - cmd_serialized: RVec, + cmd_serialized: SVec, ) -> Result { let runtime = factory.runtime().clone(); let ffi_logical_codec = factory.logical_codec.clone(); @@ -269,7 +268,7 @@ impl ForeignTableProviderFactory { fn serialize_cmd( &self, cmd: CreateExternalTable, - ) -> Result, DataFusionError> { + ) -> Result, DataFusionError> { let logical_codec: Arc = (&self.0.logical_codec).into(); @@ -280,7 +279,7 @@ impl ForeignTableProviderFactory { let mut buf: Vec = Vec::new(); plan.try_encode(&mut buf)?; - Ok(buf.into()) + Ok(buf.into_iter().collect()) } } diff --git a/datafusion/ffi/src/table_source.rs b/datafusion/ffi/src/table_source.rs index 2f17d9235a088..00374043af014 100644 --- a/datafusion/ffi/src/table_source.rs +++ b/datafusion/ffi/src/table_source.rs @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use datafusion_expr::{TableProviderFilterPushDown, TableType}; /// FFI safe version of [`TableProviderFilterPushDown`]. -#[repr(C)] -#[derive(StableAbi)] +#[expect(non_camel_case_types)] +#[repr(u8)] pub enum FFI_TableProviderFilterPushDown { Unsupported, Inexact, @@ -57,7 +56,7 @@ impl From<&TableProviderFilterPushDown> for FFI_TableProviderFilterPushDown { /// FFI safe version of [`TableType`]. #[repr(C)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, StableAbi)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FFI_TableType { Base, View, diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs index 378d75b9560c7..52e39ac6dfd04 100644 --- a/datafusion/ffi/src/tests/mod.rs +++ b/datafusion/ffi/src/tests/mod.rs @@ -17,12 +17,6 @@ use std::sync::Arc; -use abi_stable::library::{LibraryError, RootModule}; -use abi_stable::prefix_type::PrefixTypeTrait; -use abi_stable::sabi_types::VersionStrings; -use abi_stable::{ - StableAbi, declare_root_module_statics, export_root_module, package_version_strings, -}; use arrow::array::RecordBatch; use arrow_schema::{DataType, Field, Schema}; use async_provider::create_async_table_provider; @@ -57,8 +51,6 @@ mod udf_udaf_udwf; pub mod utils; #[repr(C)] -#[derive(StableAbi)] -#[sabi(kind(Prefix(prefix_ref = ForeignLibraryModuleRef)))] /// This struct defines the module interfaces. It is to be shared by /// both the module loading program and library that implements the /// module. @@ -107,17 +99,6 @@ pub struct ForeignLibraryModule { pub version: extern "C" fn() -> u64, } -impl RootModule for ForeignLibraryModuleRef { - declare_root_module_statics! {ForeignLibraryModuleRef} - const BASE_NAME: &'static str = "datafusion_ffi"; - const NAME: &'static str = "datafusion_ffi"; - const VERSION_STRINGS: VersionStrings = package_version_strings!(); - - fn initialization(self) -> Result { - Ok(self) - } -} - pub fn create_test_schema() -> Arc { Arc::new(Schema::new(vec![ Field::new("a", DataType::Int32, true), @@ -160,9 +141,9 @@ pub(crate) extern "C" fn create_empty_exec() -> FFI_ExecutionPlan { FFI_ExecutionPlan::new(plan, None) } -#[export_root_module] /// This defines the entry point for using the module. -pub fn get_foreign_library_module() -> ForeignLibraryModuleRef { +#[unsafe(no_mangle)] +pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule { ForeignLibraryModule { create_catalog: create_catalog_provider, create_catalog_list: create_catalog_provider_list, @@ -179,5 +160,4 @@ pub fn get_foreign_library_module() -> ForeignLibraryModuleRef { create_empty_exec, version: super::version, } - .leak_into_prefix() } diff --git a/datafusion/ffi/src/tests/utils.rs b/datafusion/ffi/src/tests/utils.rs index 9659a51f04b01..e1374c786266b 100644 --- a/datafusion/ffi/src/tests/utils.rs +++ b/datafusion/ffi/src/tests/utils.rs @@ -15,47 +15,63 @@ // specific language governing permissions and limitations // under the License. -use std::path::Path; +use std::path::{Path, PathBuf}; -use abi_stable::library::RootModule; use datafusion_common::{DataFusionError, Result}; -use crate::tests::ForeignLibraryModuleRef; +use crate::tests::ForeignLibraryModule; -/// Compute the path to the library. It would be preferable to simply use -/// abi_stable::library::development_utils::compute_library_path however -/// our current CI pipeline has a `ci` profile that we need to use to -/// find the library. -pub fn compute_library_path( - target_path: &Path, -) -> std::io::Result { +/// Compute the path to the built cdylib. Checks debug, release, and ci profile dirs. +fn compute_library_dir(target_path: &Path) -> PathBuf { let debug_dir = target_path.join("debug"); let release_dir = target_path.join("release"); let ci_dir = target_path.join("ci"); - let debug_path = M::get_library_path(&debug_dir.join("deps")); - let release_path = M::get_library_path(&release_dir.join("deps")); - let ci_path = M::get_library_path(&ci_dir.join("deps")); + let all_dirs = vec![debug_dir.clone(), release_dir, ci_dir]; - let all_paths = vec![ - (debug_dir.clone(), debug_path), - (release_dir, release_path), - (ci_dir, ci_path), - ]; - - let best_path = all_paths + all_dirs .into_iter() - .filter(|(_, path)| path.exists()) - .filter_map(|(dir, path)| path.metadata().map(|m| (dir, m)).ok()) - .filter_map(|(dir, meta)| meta.modified().map(|m| (dir, m)).ok()) + .filter(|dir| dir.join("deps").exists()) + .filter_map(|dir| { + dir.join("deps") + .metadata() + .and_then(|m| m.modified()) + .ok() + .map(|date| (dir, date)) + }) .max_by_key(|(_, date)| *date) .map(|(dir, _)| dir) - .unwrap_or(debug_dir); + .unwrap_or(debug_dir) +} + +/// Find the cdylib file for datafusion_ffi in the given directory. +fn find_cdylib(deps_dir: &Path) -> Result { + let lib_prefix = if cfg!(target_os = "windows") { + "" + } else { + "lib" + }; + let lib_ext = if cfg!(target_os = "macos") { + "dylib" + } else if cfg!(target_os = "windows") { + "dll" + } else { + "so" + }; - Ok(best_path) + let pattern = format!("{lib_prefix}datafusion_ffi.{lib_ext}"); + let lib_path = deps_dir.join(&pattern); + + if lib_path.exists() { + return Ok(lib_path); + } + + Err(DataFusionError::External( + format!("Could not find library at {}", lib_path.display()).into(), + )) } -pub fn get_module() -> Result { +pub fn get_module() -> Result { let expected_version = crate::version(); let crate_root = Path::new(env!("CARGO_MANIFEST_DIR")); @@ -66,24 +82,26 @@ pub fn get_module() -> Result { .expect("Failed to find workspace root") .join("target"); - // Find the location of the library. This is specific to the build environment, - // so you will need to change the approach here based on your use case. - // let target: &std::path::Path = "../../../../target/".as_ref(); - let library_path = - compute_library_path::(target_dir.as_path()) + let library_dir = compute_library_dir(target_dir.as_path()); + let lib_path = find_cdylib(&library_dir.join("deps"))?; + + // Load the library using libloading + let lib = unsafe { + libloading::Library::new(&lib_path) .map_err(|e| DataFusionError::External(Box::new(e)))? - .join("deps"); - - // Load the module - let module = ForeignLibraryModuleRef::load_from_directory(&library_path) - .map_err(|e| DataFusionError::External(Box::new(e)))?; - - assert_eq!( - module - .version() - .expect("Unable to call version on FFI module")(), - expected_version - ); + }; + + let get_module: libloading::Symbol ForeignLibraryModule> = unsafe { + lib.get(b"datafusion_ffi_get_module") + .map_err(|e| DataFusionError::External(Box::new(e)))? + }; + + let module = get_module(); + + assert_eq!((module.version)(), expected_version); + + // Leak the library to keep it loaded for the duration of the test + std::mem::forget(lib); Ok(module) } diff --git a/datafusion/ffi/src/udaf/accumulator.rs b/datafusion/ffi/src/udaf/accumulator.rs index 125b28598b433..3eb3ad339fa96 100644 --- a/datafusion/ffi/src/udaf/accumulator.rs +++ b/datafusion/ffi/src/udaf/accumulator.rs @@ -20,8 +20,6 @@ use std::ffi::c_void; use std::ops::Deref; use std::ptr::null_mut; -use abi_stable::StableAbi; -use abi_stable::std_types::{RResult, RVec}; use arrow::array::ArrayRef; use arrow::error::ArrowError; use datafusion_common::error::{DataFusionError, Result}; @@ -29,36 +27,38 @@ use datafusion_common::scalar::ScalarValue; use datafusion_expr::Accumulator; use prost::Message; +use stabby::vec::Vec as SVec; + use crate::arrow_wrappers::WrappedArray; -use crate::util::FFIResult; -use crate::{df_result, rresult, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult, sresult_return}; /// A stable struct for sharing [`Accumulator`] across FFI boundaries. /// For an explanation of each field, see the corresponding function /// defined in [`Accumulator`]. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_Accumulator { pub update_batch: unsafe extern "C" fn( accumulator: &mut Self, - values: RVec, + values: SVec, ) -> FFIResult<()>, // Evaluate and return a ScalarValues as protobuf bytes - pub evaluate: unsafe extern "C" fn(accumulator: &mut Self) -> FFIResult>, + pub evaluate: unsafe extern "C" fn(accumulator: &mut Self) -> FFIResult>, pub size: unsafe extern "C" fn(accumulator: &Self) -> usize, - pub state: unsafe extern "C" fn(accumulator: &mut Self) -> FFIResult>>, + pub state: unsafe extern "C" fn(accumulator: &mut Self) -> FFIResult>>, pub merge_batch: unsafe extern "C" fn( accumulator: &mut Self, - states: RVec, + states: SVec, ) -> FFIResult<()>, pub retract_batch: unsafe extern "C" fn( accumulator: &mut Self, - values: RVec, + values: SVec, ) -> FFIResult<()>, pub supports_retract_batch: bool, @@ -103,7 +103,7 @@ impl FFI_Accumulator { unsafe extern "C" fn update_batch_fn_wrapper( accumulator: &mut FFI_Accumulator, - values: RVec, + values: SVec, ) -> FFIResult<()> { unsafe { let accumulator = accumulator.inner_mut(); @@ -112,23 +112,23 @@ unsafe extern "C" fn update_batch_fn_wrapper( .into_iter() .map(|v| v.try_into().map_err(DataFusionError::from)) .collect::>>(); - let values_arrays = rresult_return!(values_arrays); + let values_arrays = sresult_return!(values_arrays); - rresult!(accumulator.update_batch(&values_arrays)) + sresult!(accumulator.update_batch(&values_arrays)) } } unsafe extern "C" fn evaluate_fn_wrapper( accumulator: &mut FFI_Accumulator, -) -> FFIResult> { +) -> FFIResult> { unsafe { let accumulator = accumulator.inner_mut(); - let scalar_result = rresult_return!(accumulator.evaluate()); + let scalar_result = sresult_return!(accumulator.evaluate()); let proto_result: datafusion_proto::protobuf::ScalarValue = - rresult_return!((&scalar_result).try_into()); + sresult_return!((&scalar_result).try_into()); - RResult::ROk(proto_result.encode_to_vec().into()) + FFI_Result::Ok(proto_result.encode_to_vec().into_iter().collect()) } } @@ -138,46 +138,46 @@ unsafe extern "C" fn size_fn_wrapper(accumulator: &FFI_Accumulator) -> usize { unsafe extern "C" fn state_fn_wrapper( accumulator: &mut FFI_Accumulator, -) -> FFIResult>> { +) -> FFIResult>> { unsafe { let accumulator = accumulator.inner_mut(); - let state = rresult_return!(accumulator.state()); + let state = sresult_return!(accumulator.state()); let state = state .into_iter() .map(|state_val| { datafusion_proto::protobuf::ScalarValue::try_from(&state_val) .map_err(DataFusionError::from) - .map(|v| RVec::from(v.encode_to_vec())) + .map(|v| v.encode_to_vec().into_iter().collect::>()) }) .collect::>>() - .map(|state_vec| state_vec.into()); + .map(|state_vec| state_vec.into_iter().collect()); - rresult!(state) + sresult!(state) } } unsafe extern "C" fn merge_batch_fn_wrapper( accumulator: &mut FFI_Accumulator, - states: RVec, + states: SVec, ) -> FFIResult<()> { unsafe { let accumulator = accumulator.inner_mut(); - let states = rresult_return!( + let states = sresult_return!( states .into_iter() .map(|state| ArrayRef::try_from(state).map_err(DataFusionError::from)) .collect::>>() ); - rresult!(accumulator.merge_batch(&states)) + sresult!(accumulator.merge_batch(&states)) } } unsafe extern "C" fn retract_batch_fn_wrapper( accumulator: &mut FFI_Accumulator, - values: RVec, + values: SVec, ) -> FFIResult<()> { unsafe { let accumulator = accumulator.inner_mut(); @@ -186,9 +186,9 @@ unsafe extern "C" fn retract_batch_fn_wrapper( .into_iter() .map(|v| v.try_into().map_err(DataFusionError::from)) .collect::>>(); - let values_arrays = rresult_return!(values_arrays); + let values_arrays = sresult_return!(values_arrays); - rresult!(accumulator.retract_batch(&values_arrays)) + sresult!(accumulator.retract_batch(&values_arrays)) } } @@ -273,7 +273,7 @@ impl Accumulator for ForeignAccumulator { .collect::, ArrowError>>()?; df_result!((self.accumulator.update_batch)( &mut self.accumulator, - values.into() + values.into_iter().collect() )) } } @@ -322,7 +322,7 @@ impl Accumulator for ForeignAccumulator { .collect::, ArrowError>>()?; df_result!((self.accumulator.merge_batch)( &mut self.accumulator, - states.into() + states.into_iter().collect() )) } } @@ -335,7 +335,7 @@ impl Accumulator for ForeignAccumulator { .collect::, ArrowError>>()?; df_result!((self.accumulator.retract_batch)( &mut self.accumulator, - values.into() + values.into_iter().collect() )) } } diff --git a/datafusion/ffi/src/udaf/accumulator_args.rs b/datafusion/ffi/src/udaf/accumulator_args.rs index a3359231073c4..58bcf58b88c2b 100644 --- a/datafusion/ffi/src/udaf/accumulator_args.rs +++ b/datafusion/ffi/src/udaf/accumulator_args.rs @@ -17,14 +17,14 @@ use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{RString, RVec}; use arrow::datatypes::Schema; use arrow::ffi::FFI_ArrowSchema; use arrow_schema::FieldRef; use datafusion_common::error::DataFusionError; use datafusion_expr::function::AccumulatorArgs; use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr}; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use crate::arrow_wrappers::WrappedSchema; use crate::physical_expr::FFI_PhysicalExpr; @@ -35,17 +35,17 @@ use crate::util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped}; /// For an explanation of each field, see the corresponding field /// defined in [`AccumulatorArgs`]. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_AccumulatorArgs { return_field: WrappedSchema, schema: WrappedSchema, ignore_nulls: bool, - order_bys: RVec, + order_bys: SVec, is_reversed: bool, - name: RString, + name: SString, is_distinct: bool, - exprs: RVec, - expr_fields: RVec, + exprs: SVec, + expr_fields: SVec, } impl TryFrom> for FFI_AccumulatorArgs { @@ -55,7 +55,7 @@ impl TryFrom> for FFI_AccumulatorArgs { WrappedSchema(FFI_ArrowSchema::try_from(args.return_field.as_ref())?); let schema = WrappedSchema(FFI_ArrowSchema::try_from(args.schema)?); - let order_bys: RVec<_> = args + let order_bys: SVec<_> = args .order_bys .iter() .map(FFI_PhysicalSortExpr::from) diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rs index 0dc8edbfe5a85..ddc851a204c6e 100644 --- a/datafusion/ffi/src/udaf/groups_accumulator.rs +++ b/datafusion/ffi/src/udaf/groups_accumulator.rs @@ -21,29 +21,29 @@ use std::ops::Deref; use std::ptr::null_mut; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RVec}; use arrow::array::{Array, ArrayRef, BooleanArray}; use arrow::error::ArrowError; use arrow::ffi::to_ffi; use datafusion_common::error::{DataFusionError, Result}; use datafusion_expr::{EmitTo, GroupsAccumulator}; +use stabby::vec::Vec as SVec; + use crate::arrow_wrappers::{WrappedArray, WrappedSchema}; -use crate::util::FFIResult; -use crate::{df_result, rresult, rresult_return}; +use crate::util::{FFI_Option, FFIResult}; +use crate::{df_result, sresult, sresult_return}; /// A stable struct for sharing [`GroupsAccumulator`] across FFI boundaries. /// For an explanation of each field, see the corresponding function /// defined in [`GroupsAccumulator`]. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_GroupsAccumulator { pub update_batch: unsafe extern "C" fn( accumulator: &mut Self, - values: RVec, - group_indices: RVec, - opt_filter: ROption, + values: SVec, + group_indices: SVec, + opt_filter: FFI_Option, total_num_groups: usize, ) -> FFIResult<()>, @@ -58,21 +58,21 @@ pub struct FFI_GroupsAccumulator { pub state: unsafe extern "C" fn( accumulator: &mut Self, emit_to: FFI_EmitTo, - ) -> FFIResult>, + ) -> FFIResult>, pub merge_batch: unsafe extern "C" fn( accumulator: &mut Self, - values: RVec, - group_indices: RVec, - opt_filter: ROption, + values: SVec, + group_indices: SVec, + opt_filter: FFI_Option, total_num_groups: usize, ) -> FFIResult<()>, pub convert_to_state: unsafe extern "C" fn( accumulator: &Self, - values: RVec, - opt_filter: ROption, - ) -> FFIResult>, + values: SVec, + opt_filter: FFI_Option, + ) -> FFIResult>, pub supports_convert_to_state: bool, @@ -111,7 +111,7 @@ impl FFI_GroupsAccumulator { } } -fn process_values(values: RVec) -> Result>> { +fn process_values(values: SVec) -> Result>> { values .into_iter() .map(|v| v.try_into().map_err(DataFusionError::from)) @@ -119,7 +119,9 @@ fn process_values(values: RVec) -> Result>> { } /// Convert C-typed opt_filter into the internal type. -fn process_opt_filter(opt_filter: ROption) -> Result> { +fn process_opt_filter( + opt_filter: FFI_Option, +) -> Result> { opt_filter .into_option() .map(|filter| { @@ -132,18 +134,18 @@ fn process_opt_filter(opt_filter: ROption) -> Result, - group_indices: RVec, - opt_filter: ROption, + values: SVec, + group_indices: SVec, + opt_filter: FFI_Option, total_num_groups: usize, ) -> FFIResult<()> { unsafe { let accumulator = accumulator.inner_mut(); - let values = rresult_return!(process_values(values)); + let values = sresult_return!(process_values(values)); let group_indices: Vec = group_indices.into_iter().collect(); - let opt_filter = rresult_return!(process_opt_filter(opt_filter)); + let opt_filter = sresult_return!(process_opt_filter(opt_filter)); - rresult!(accumulator.update_batch( + sresult!(accumulator.update_batch( &values, &group_indices, opt_filter.as_ref(), @@ -159,9 +161,9 @@ unsafe extern "C" fn evaluate_fn_wrapper( unsafe { let accumulator = accumulator.inner_mut(); - let result = rresult_return!(accumulator.evaluate(emit_to.into())); + let result = sresult_return!(accumulator.evaluate(emit_to.into())); - rresult!(WrappedArray::try_from(&result)) + sresult!(WrappedArray::try_from(&result)) } } @@ -175,34 +177,34 @@ unsafe extern "C" fn size_fn_wrapper(accumulator: &FFI_GroupsAccumulator) -> usi unsafe extern "C" fn state_fn_wrapper( accumulator: &mut FFI_GroupsAccumulator, emit_to: FFI_EmitTo, -) -> FFIResult> { +) -> FFIResult> { unsafe { let accumulator = accumulator.inner_mut(); - let state = rresult_return!(accumulator.state(emit_to.into())); - rresult!( + let state = sresult_return!(accumulator.state(emit_to.into())); + sresult!( state .into_iter() .map(|arr| WrappedArray::try_from(&arr).map_err(DataFusionError::from)) - .collect::>>() + .collect::>>() ) } } unsafe extern "C" fn merge_batch_fn_wrapper( accumulator: &mut FFI_GroupsAccumulator, - values: RVec, - group_indices: RVec, - opt_filter: ROption, + values: SVec, + group_indices: SVec, + opt_filter: FFI_Option, total_num_groups: usize, ) -> FFIResult<()> { unsafe { let accumulator = accumulator.inner_mut(); - let values = rresult_return!(process_values(values)); + let values = sresult_return!(process_values(values)); let group_indices: Vec = group_indices.into_iter().collect(); - let opt_filter = rresult_return!(process_opt_filter(opt_filter)); + let opt_filter = sresult_return!(process_opt_filter(opt_filter)); - rresult!(accumulator.merge_batch( + sresult!(accumulator.merge_batch( &values, &group_indices, opt_filter.as_ref(), @@ -213,21 +215,21 @@ unsafe extern "C" fn merge_batch_fn_wrapper( unsafe extern "C" fn convert_to_state_fn_wrapper( accumulator: &FFI_GroupsAccumulator, - values: RVec, - opt_filter: ROption, -) -> FFIResult> { + values: SVec, + opt_filter: FFI_Option, +) -> FFIResult> { unsafe { let accumulator = accumulator.inner(); - let values = rresult_return!(process_values(values)); - let opt_filter = rresult_return!(process_opt_filter(opt_filter)); + let values = sresult_return!(process_values(values)); + let opt_filter = sresult_return!(process_opt_filter(opt_filter)); let state = - rresult_return!(accumulator.convert_to_state(&values, opt_filter.as_ref())); + sresult_return!(accumulator.convert_to_state(&values, opt_filter.as_ref())); - rresult!( + sresult!( state .iter() .map(|arr| WrappedArray::try_from(arr).map_err(DataFusionError::from)) - .collect::>>() + .collect::>>() ) } } @@ -334,7 +336,7 @@ impl GroupsAccumulator for ForeignGroupsAccumulator { df_result!((self.accumulator.update_batch)( &mut self.accumulator, - values.into(), + values.into_iter().collect(), group_indices, opt_filter, total_num_groups @@ -397,7 +399,7 @@ impl GroupsAccumulator for ForeignGroupsAccumulator { df_result!((self.accumulator.merge_batch)( &mut self.accumulator, - values.into(), + values.into_iter().collect(), group_indices, opt_filter, total_num_groups @@ -414,7 +416,7 @@ impl GroupsAccumulator for ForeignGroupsAccumulator { let values = values .iter() .map(WrappedArray::try_from) - .collect::, ArrowError>>()?; + .collect::, ArrowError>>()?; let opt_filter = opt_filter .map(|bool_array| to_ffi(&bool_array.to_data())) @@ -444,7 +446,7 @@ impl GroupsAccumulator for ForeignGroupsAccumulator { } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub enum FFI_EmitTo { All, First(usize), diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs index 8e791b28b1ad6..05f5f11299a6e 100644 --- a/datafusion/ffi/src/udaf/mod.rs +++ b/datafusion/ffi/src/udaf/mod.rs @@ -15,12 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::c_void; -use std::hash::{Hash, Hasher}; -use std::sync::Arc; - -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RStr, RString, RVec}; use accumulator::FFI_Accumulator; use accumulator_args::{FFI_AccumulatorArgs, ForeignAccumulatorArgs}; use arrow::datatypes::{DataType, Field}; @@ -40,13 +34,21 @@ use datafusion_proto_common::from_proto::parse_proto_fields_to_fields; use groups_accumulator::FFI_GroupsAccumulator; use prost::{DecodeError, Message}; +use stabby::str::Str as SStr; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; +use std::ffi::c_void; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + use crate::arrow_wrappers::WrappedSchema; use crate::util::{ - FFIResult, rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref, - vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped, + FFI_Option, FFI_Result, FFIResult, rvec_wrapped_to_vec_datatype, + rvec_wrapped_to_vec_fieldref, vec_datatype_to_rvec_wrapped, + vec_fieldref_to_rvec_wrapped, }; use crate::volatility::FFI_Volatility; -use crate::{df_result, rresult, rresult_return}; +use crate::{df_result, sresult, sresult_return}; mod accumulator; mod accumulator_args; @@ -54,13 +56,13 @@ mod groups_accumulator; /// A stable struct for sharing a [`AggregateUDF`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_AggregateUDF { /// FFI equivalent to the `name` of a [`AggregateUDF`] - pub name: RString, + pub name: SString, /// FFI equivalent to the `aliases` of a [`AggregateUDF`] - pub aliases: RVec, + pub aliases: SVec, /// FFI equivalent to the `volatility` of a [`AggregateUDF`] pub volatility: FFI_Volatility, @@ -69,7 +71,7 @@ pub struct FFI_AggregateUDF { /// argument fields. pub return_field: unsafe extern "C" fn( udaf: &Self, - arg_fields: RVec, + arg_fields: SVec, ) -> FFIResult, /// FFI equivalent to the `is_nullable` of a [`AggregateUDF`] @@ -95,12 +97,12 @@ pub struct FFI_AggregateUDF { /// FFI equivalent to [`AggregateUDF::state_fields`] pub state_fields: unsafe extern "C" fn( udaf: &FFI_AggregateUDF, - name: &RStr, - input_fields: RVec, + name: &SStr, + input_fields: SVec, return_field: WrappedSchema, - ordering_fields: RVec>, + ordering_fields: SVec>, is_distinct: bool, - ) -> FFIResult>>, + ) -> FFIResult>>, /// FFI equivalent to [`AggregateUDF::create_groups_accumulator`] pub create_groups_accumulator: @@ -114,7 +116,7 @@ pub struct FFI_AggregateUDF { unsafe extern "C" fn( udaf: &FFI_AggregateUDF, beneficial_ordering: bool, - ) -> FFIResult>, + ) -> FFIResult>, /// FFI equivalent to [`AggregateUDF::order_sensitivity`] pub order_sensitivity: @@ -126,8 +128,8 @@ pub struct FFI_AggregateUDF { /// appropriate calls on the underlying [`AggregateUDF`] pub coerce_types: unsafe extern "C" fn( udf: &Self, - arg_types: RVec, - ) -> FFIResult>, + arg_types: SVec, + ) -> FFIResult>, /// Used to create a clone on the provider of the udaf. This should /// only need to be called by the receiver of the udaf. @@ -164,12 +166,12 @@ impl FFI_AggregateUDF { unsafe extern "C" fn return_field_fn_wrapper( udaf: &FFI_AggregateUDF, - arg_fields: RVec, + arg_fields: SVec, ) -> FFIResult { unsafe { let udaf = udaf.inner(); - let arg_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&arg_fields)); + let arg_fields = sresult_return!(rvec_wrapped_to_vec_fieldref(&arg_fields)); let return_field = udaf .return_field(&arg_fields) @@ -178,7 +180,7 @@ unsafe extern "C" fn return_field_fn_wrapper( }) .map(WrappedSchema); - rresult!(return_field) + sresult!(return_field) } } @@ -189,9 +191,9 @@ unsafe extern "C" fn accumulator_fn_wrapper( unsafe { let udaf = udaf.inner(); - let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args)); + let accumulator_args = &sresult_return!(ForeignAccumulatorArgs::try_from(args)); - rresult!( + sresult!( udaf.accumulator(accumulator_args.into()) .map(FFI_Accumulator::from) ) @@ -205,9 +207,9 @@ unsafe extern "C" fn create_sliding_accumulator_fn_wrapper( unsafe { let udaf = udaf.inner(); - let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args)); + let accumulator_args = &sresult_return!(ForeignAccumulatorArgs::try_from(args)); - rresult!( + sresult!( udaf.create_sliding_accumulator(accumulator_args.into()) .map(FFI_Accumulator::from) ) @@ -221,9 +223,9 @@ unsafe extern "C" fn create_groups_accumulator_fn_wrapper( unsafe { let udaf = udaf.inner(); - let accumulator_args = &rresult_return!(ForeignAccumulatorArgs::try_from(args)); + let accumulator_args = &sresult_return!(ForeignAccumulatorArgs::try_from(args)); - rresult!( + sresult!( udaf.create_groups_accumulator(accumulator_args.into()) .map(FFI_GroupsAccumulator::from) ) @@ -249,12 +251,12 @@ unsafe extern "C" fn groups_accumulator_supported_fn_wrapper( unsafe extern "C" fn with_beneficial_ordering_fn_wrapper( udaf: &FFI_AggregateUDF, beneficial_ordering: bool, -) -> FFIResult> { +) -> FFIResult> { unsafe { let udaf = udaf.inner().as_ref().clone(); - let result = rresult_return!(udaf.with_beneficial_ordering(beneficial_ordering)); - let result = rresult_return!( + let result = sresult_return!(udaf.with_beneficial_ordering(beneficial_ordering)); + let result = sresult_return!( result .map(|func| func.with_beneficial_ordering(beneficial_ordering)) .transpose() @@ -262,25 +264,25 @@ unsafe extern "C" fn with_beneficial_ordering_fn_wrapper( .flatten() .map(|func| FFI_AggregateUDF::from(Arc::new(func))); - RResult::ROk(result.into()) + FFI_Result::Ok(result.into()) } } unsafe extern "C" fn state_fields_fn_wrapper( udaf: &FFI_AggregateUDF, - name: &RStr, - input_fields: RVec, + name: &SStr, + input_fields: SVec, return_field: WrappedSchema, - ordering_fields: RVec>, + ordering_fields: SVec>, is_distinct: bool, -) -> FFIResult>> { +) -> FFIResult>> { unsafe { let udaf = udaf.inner(); - let input_fields = &rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields)); - let return_field = rresult_return!(Field::try_from(&return_field.0)).into(); + let input_fields = &sresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields)); + let return_field = sresult_return!(Field::try_from(&return_field.0)).into(); - let ordering_fields = &rresult_return!( + let ordering_fields = &sresult_return!( ordering_fields .into_iter() .map(|field_bytes| datafusion_proto_common::Field::decode( @@ -290,7 +292,7 @@ unsafe extern "C" fn state_fields_fn_wrapper( ); let ordering_fields = - &rresult_return!(parse_proto_fields_to_fields(ordering_fields)) + &sresult_return!(parse_proto_fields_to_fields(ordering_fields)) .into_iter() .map(Arc::new) .collect::>(); @@ -303,8 +305,8 @@ unsafe extern "C" fn state_fields_fn_wrapper( is_distinct, }; - let state_fields = rresult_return!(udaf.state_fields(args)); - let state_fields = rresult_return!( + let state_fields = sresult_return!(udaf.state_fields(args)); + let state_fields = sresult_return!( state_fields .iter() .map(|f| f.as_ref()) @@ -313,10 +315,10 @@ unsafe extern "C" fn state_fields_fn_wrapper( .collect::>>() ) .into_iter() - .map(|field| field.encode_to_vec().into()) + .map(|field| field.encode_to_vec().into_iter().collect()) .collect(); - RResult::ROk(state_fields) + FFI_Result::Ok(state_fields) } } @@ -328,24 +330,24 @@ unsafe extern "C" fn order_sensitivity_fn_wrapper( unsafe extern "C" fn coerce_types_fn_wrapper( udaf: &FFI_AggregateUDF, - arg_types: RVec, -) -> FFIResult> { + arg_types: SVec, +) -> FFIResult> { unsafe { let udaf = udaf.inner(); - let arg_types = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types)); + let arg_types = sresult_return!(rvec_wrapped_to_vec_datatype(&arg_types)); let arg_fields = arg_types .iter() .map(|dt| Field::new("f", dt.clone(), true)) .map(Arc::new) .collect::>(); - let return_types = rresult_return!(fields_with_udf(&arg_fields, udaf.as_ref())) + let return_types = sresult_return!(fields_with_udf(&arg_fields, udaf.as_ref())) .into_iter() .map(|f| f.data_type().to_owned()) .collect::>(); - rresult!(vec_datatype_to_rvec_wrapped(&return_types)) + sresult!(vec_datatype_to_rvec_wrapped(&return_types)) } } @@ -501,7 +503,7 @@ impl AggregateUDFImpl for ForeignAggregateUDF { fn state_fields(&self, args: StateFieldsArgs) -> Result> { unsafe { - let name = RStr::from_str(args.name); + let name = SStr::from(args.name); let input_fields = vec_fieldref_to_rvec_wrapped(args.input_fields)?; let return_field = WrappedSchema(FFI_ArrowSchema::try_from(args.return_field.as_ref())?); @@ -513,7 +515,7 @@ impl AggregateUDFImpl for ForeignAggregateUDF { .map(|v| v.map_err(DataFusionError::from)) .collect::>>()? .into_iter() - .map(|proto_field| proto_field.encode_to_vec().into()) + .map(|proto_field| proto_field.encode_to_vec().into_iter().collect()) .collect(); let fields = df_result!((self.udaf.state_fields)( @@ -613,7 +615,7 @@ impl AggregateUDFImpl for ForeignAggregateUDF { } #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub enum FFI_AggregateOrderSensitivity { Insensitive, HardRequirement, diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rs index 0202fd8bcfe65..a2f2691e76de5 100644 --- a/datafusion/ffi/src/udf/mod.rs +++ b/datafusion/ffi/src/udf/mod.rs @@ -19,8 +19,6 @@ use std::ffi::c_void; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{RString, RVec}; use arrow::array::Array; use arrow::datatypes::{DataType, Field}; use arrow::error::ArrowError; @@ -37,6 +35,9 @@ use return_type_args::{ FFI_ReturnFieldArgs, ForeignReturnFieldArgs, ForeignReturnFieldArgsOwned, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; + use crate::arrow_wrappers::{WrappedArray, WrappedSchema}; use crate::config::FFI_ConfigOptions; use crate::expr::columnar_value::FFI_ColumnarValue; @@ -44,19 +45,19 @@ use crate::util::{ FFIResult, rvec_wrapped_to_vec_datatype, vec_datatype_to_rvec_wrapped, }; use crate::volatility::FFI_Volatility; -use crate::{df_result, rresult, rresult_return}; +use crate::{df_result, sresult, sresult_return}; pub mod return_type_args; /// A stable struct for sharing a [`ScalarUDF`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_ScalarUDF { /// FFI equivalent to the `name` of a [`ScalarUDF`] - pub name: RString, + pub name: SString, /// FFI equivalent to the `aliases` of a [`ScalarUDF`] - pub aliases: RVec, + pub aliases: SVec, /// FFI equivalent to the `volatility` of a [`ScalarUDF`] pub volatility: FFI_Volatility, @@ -71,8 +72,8 @@ pub struct FFI_ScalarUDF { /// within an AbiStable wrapper. pub invoke_with_args: unsafe extern "C" fn( udf: &Self, - args: RVec, - arg_fields: RVec, + args: SVec, + arg_fields: SVec, num_rows: usize, return_field: WrappedSchema, config_options: FFI_ConfigOptions, @@ -87,8 +88,8 @@ pub struct FFI_ScalarUDF { /// appropriate calls on the underlying [`ScalarUDF`] pub coerce_types: unsafe extern "C" fn( udf: &Self, - arg_types: RVec, - ) -> FFIResult>, + arg_types: SVec, + ) -> FFIResult>, /// Used to create a clone on the provider of the udf. This should /// only need to be called by the receiver of the udf. @@ -125,7 +126,7 @@ unsafe extern "C" fn return_field_from_args_fn_wrapper( udf: &FFI_ScalarUDF, args: FFI_ReturnFieldArgs, ) -> FFIResult { - let args: ForeignReturnFieldArgsOwned = rresult_return!((&args).try_into()); + let args: ForeignReturnFieldArgsOwned = sresult_return!((&args).try_into()); let args_ref: ForeignReturnFieldArgs = (&args).into(); let return_type = udf @@ -134,32 +135,32 @@ unsafe extern "C" fn return_field_from_args_fn_wrapper( .and_then(|f| FFI_ArrowSchema::try_from(&f).map_err(DataFusionError::from)) .map(WrappedSchema); - rresult!(return_type) + sresult!(return_type) } unsafe extern "C" fn coerce_types_fn_wrapper( udf: &FFI_ScalarUDF, - arg_types: RVec, -) -> FFIResult> { - let arg_types = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types)); + arg_types: SVec, +) -> FFIResult> { + let arg_types = sresult_return!(rvec_wrapped_to_vec_datatype(&arg_types)); let arg_fields = arg_types .iter() .map(|dt| Arc::new(Field::new("f", dt.clone(), true))) .collect::>(); let return_types = - rresult_return!(fields_with_udf(&arg_fields, udf.inner().as_ref())) + sresult_return!(fields_with_udf(&arg_fields, udf.inner().as_ref())) .into_iter() .map(|f| f.data_type().to_owned()) .collect::>(); - rresult!(vec_datatype_to_rvec_wrapped(&return_types)) + sresult!(vec_datatype_to_rvec_wrapped(&return_types)) } unsafe extern "C" fn invoke_with_args_fn_wrapper( udf: &FFI_ScalarUDF, - args: RVec, - arg_fields: RVec, + args: SVec, + arg_fields: SVec, number_rows: usize, return_field: WrappedSchema, config_options: FFI_ConfigOptions, @@ -173,8 +174,8 @@ unsafe extern "C" fn invoke_with_args_fn_wrapper( }) .collect::>(); - let args = rresult_return!(args); - let return_field = rresult_return!(Field::try_from(&return_field.0)).into(); + let args = sresult_return!(args); + let return_field = sresult_return!(Field::try_from(&return_field.0)).into(); let arg_fields = arg_fields .into_iter() @@ -184,8 +185,8 @@ unsafe extern "C" fn invoke_with_args_fn_wrapper( .map_err(DataFusionError::from) }) .collect::>>(); - let arg_fields = rresult_return!(arg_fields); - let config_options = rresult_return!(ConfigOptions::try_from(config_options)); + let arg_fields = sresult_return!(arg_fields); + let config_options = sresult_return!(ConfigOptions::try_from(config_options)); let config_options = Arc::new(config_options); let args = ScalarFunctionArgs { @@ -196,7 +197,7 @@ unsafe extern "C" fn invoke_with_args_fn_wrapper( config_options, }; - rresult!( + sresult!( udf.inner() .invoke_with_args(args) .and_then(FFI_ColumnarValue::try_from) @@ -316,7 +317,7 @@ impl From<&FFI_ScalarUDF> for Arc { if (udf.library_marker_id)() == crate::get_library_marker_id() { Arc::clone(udf.inner().inner()) } else { - let name = udf.name.to_owned().into(); + let name = udf.name.to_string(); let signature = Signature::user_defined((&udf.volatility).into()); let aliases = udf.aliases.iter().map(|s| s.to_string()).collect(); @@ -383,7 +384,8 @@ impl ScalarUDFImpl for ForeignScalarUDF { }) }) .collect::, ArrowError>>()? - .into(); + .into_iter() + .collect(); let arg_fields_wrapped = arg_fields .iter() @@ -393,7 +395,7 @@ impl ScalarUDFImpl for ForeignScalarUDF { let arg_fields = arg_fields_wrapped .into_iter() .map(WrappedSchema) - .collect::>(); + .collect::>(); let return_field = return_field.as_ref().clone(); let return_field = WrappedSchema(FFI_ArrowSchema::try_from(return_field)?); diff --git a/datafusion/ffi/src/udf/return_type_args.rs b/datafusion/ffi/src/udf/return_type_args.rs index 8fb015b7ed922..5a7cec8310c77 100644 --- a/datafusion/ffi/src/udf/return_type_args.rs +++ b/datafusion/ffi/src/udf/return_type_args.rs @@ -15,23 +15,25 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RVec}; use arrow_schema::FieldRef; use datafusion_common::scalar::ScalarValue; use datafusion_common::{DataFusionError, ffi_datafusion_err}; use datafusion_expr::ReturnFieldArgs; use prost::Message; +use stabby::vec::Vec as SVec; + use crate::arrow_wrappers::WrappedSchema; -use crate::util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped}; +use crate::util::{ + FFI_Option, rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped, +}; /// A stable struct for sharing a [`ReturnFieldArgs`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_ReturnFieldArgs { - arg_fields: RVec, - scalar_arguments: RVec>>, + arg_fields: SVec, + scalar_arguments: SVec>>, } impl TryFrom> for FFI_ReturnFieldArgs { @@ -47,13 +49,17 @@ impl TryFrom> for FFI_ReturnFieldArgs { .map(|arg| { let proto_value: datafusion_proto::protobuf::ScalarValue = arg.try_into()?; - let proto_bytes: RVec = proto_value.encode_to_vec().into(); + let proto_bytes: SVec = + proto_value.encode_to_vec().into_iter().collect(); Ok(proto_bytes) }) .transpose() }) .collect(); - let scalar_arguments = scalar_arguments?.into_iter().map(ROption::from).collect(); + let scalar_arguments = scalar_arguments? + .into_iter() + .map(FFI_Option::from) + .collect(); Ok(Self { arg_fields, @@ -91,7 +97,7 @@ impl TryFrom<&FFI_ReturnFieldArgs> for ForeignReturnFieldArgsOwned { let scalar_value: ScalarValue = (&proto_value).try_into()?; Ok(scalar_value) }); - Option::from(maybe_arg).transpose() + maybe_arg.transpose() }) .collect(); let scalar_arguments = scalar_arguments?.into_iter().collect(); diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rs index 35c13c1169c72..cfd0fec9125bd 100644 --- a/datafusion/ffi/src/udtf.rs +++ b/datafusion/ffi/src/udtf.rs @@ -19,8 +19,6 @@ use std::any::Any; use std::ffi::c_void; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{RResult, RVec}; use datafusion_catalog::{TableFunctionImpl, TableProvider}; use datafusion_common::error::Result; use datafusion_execution::TaskContext; @@ -32,22 +30,24 @@ use datafusion_proto::logical_plan::{ }; use datafusion_proto::protobuf::LogicalExprList; use prost::Message; + +use stabby::vec::Vec as SVec; use tokio::runtime::Handle; use crate::execution::FFI_TaskContextProvider; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use crate::table_provider::FFI_TableProvider; -use crate::util::FFIResult; -use crate::{df_result, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult_return}; /// A stable struct for sharing a [`TableFunctionImpl`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_TableFunction { /// Equivalent to the `call` function of the TableFunctionImpl. /// The arguments are Expr passed as protobuf encoded bytes. pub call: - unsafe extern "C" fn(udtf: &Self, args: RVec) -> FFIResult, + unsafe extern "C" fn(udtf: &Self, args: SVec) -> FFIResult, pub logical_codec: FFI_LogicalExtensionCodec, @@ -90,25 +90,25 @@ impl FFI_TableFunction { unsafe extern "C" fn call_fn_wrapper( udtf: &FFI_TableFunction, - args: RVec, + args: SVec, ) -> FFIResult { let runtime = udtf.runtime(); let udtf_inner = udtf.inner(); let ctx: Arc = - rresult_return!((&udtf.logical_codec.task_ctx_provider).try_into()); + sresult_return!((&udtf.logical_codec.task_ctx_provider).try_into()); let codec: Arc = (&udtf.logical_codec).into(); - let proto_filters = rresult_return!(LogicalExprList::decode(args.as_ref())); + let proto_filters = sresult_return!(LogicalExprList::decode(args.as_ref())); - let args = rresult_return!(parse_exprs( + let args = sresult_return!(parse_exprs( proto_filters.expr.iter(), ctx.as_ref(), codec.as_ref() )); - let table_provider = rresult_return!(udtf_inner.call(&args)); - RResult::ROk(FFI_TableProvider::new_with_ffi_codec( + let table_provider = sresult_return!(udtf_inner.call(&args)); + FFI_Result::Ok(FFI_TableProvider::new_with_ffi_codec( table_provider, false, runtime, @@ -220,7 +220,7 @@ impl TableFunctionImpl for ForeignTableFunction { let expr_list = LogicalExprList { expr: serialize_exprs(args, codec.as_ref())?, }; - let filters_serialized = expr_list.encode_to_vec().into(); + let filters_serialized = expr_list.encode_to_vec().into_iter().collect(); let table_provider = unsafe { (self.0.call)(&self.0, filters_serialized) }; diff --git a/datafusion/ffi/src/udwf/mod.rs b/datafusion/ffi/src/udwf/mod.rs index 2e4bd0a294fd0..5ed60aa9b2d74 100644 --- a/datafusion/ffi/src/udwf/mod.rs +++ b/datafusion/ffi/src/udwf/mod.rs @@ -19,8 +19,6 @@ use std::ffi::c_void; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::{ROption, RResult, RString, RVec}; use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Schema, SchemaRef}; use arrow_schema::{Field, FieldRef}; @@ -36,27 +34,31 @@ use partition_evaluator_args::{ FFI_PartitionEvaluatorArgs, ForeignPartitionEvaluatorArgs, }; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; + mod partition_evaluator; mod partition_evaluator_args; mod range; use crate::arrow_wrappers::WrappedSchema; use crate::util::{ - FFIResult, rvec_wrapped_to_vec_datatype, rvec_wrapped_to_vec_fieldref, - vec_datatype_to_rvec_wrapped, vec_fieldref_to_rvec_wrapped, + FFI_Option, FFI_Result, FFIResult, rvec_wrapped_to_vec_datatype, + rvec_wrapped_to_vec_fieldref, vec_datatype_to_rvec_wrapped, + vec_fieldref_to_rvec_wrapped, }; use crate::volatility::FFI_Volatility; -use crate::{df_result, rresult, rresult_return}; +use crate::{df_result, sresult, sresult_return}; /// A stable struct for sharing a [`WindowUDF`] across FFI boundaries. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_WindowUDF { /// FFI equivalent to the `name` of a [`WindowUDF`] - pub name: RString, + pub name: SString, /// FFI equivalent to the `aliases` of a [`WindowUDF`] - pub aliases: RVec, + pub aliases: SVec, /// FFI equivalent to the `volatility` of a [`WindowUDF`] pub volatility: FFI_Volatility, @@ -69,8 +71,8 @@ pub struct FFI_WindowUDF { pub field: unsafe extern "C" fn( udwf: &Self, - input_types: RVec, - display_name: RString, + input_types: SVec, + display_name: SString, ) -> FFIResult, /// Performs type coercion. To simply this interface, all UDFs are treated as having @@ -79,10 +81,10 @@ pub struct FFI_WindowUDF { /// appropriate calls on the underlying [`WindowUDF`] pub coerce_types: unsafe extern "C" fn( udf: &Self, - arg_types: RVec, - ) -> FFIResult>, + arg_types: SVec, + ) -> FFIResult>, - pub sort_options: ROption, + pub sort_options: FFI_Option, /// Used to create a clone on the provider of the udf. This should /// only need to be called by the receiver of the udf. @@ -124,56 +126,56 @@ unsafe extern "C" fn partition_evaluator_fn_wrapper( unsafe { let inner = udwf.inner(); - let args = rresult_return!(ForeignPartitionEvaluatorArgs::try_from(args)); + let args = sresult_return!(ForeignPartitionEvaluatorArgs::try_from(args)); let evaluator = - rresult_return!(inner.partition_evaluator_factory((&args).into())); + sresult_return!(inner.partition_evaluator_factory((&args).into())); - RResult::ROk(evaluator.into()) + FFI_Result::Ok(evaluator.into()) } } unsafe extern "C" fn field_fn_wrapper( udwf: &FFI_WindowUDF, - input_fields: RVec, - display_name: RString, + input_fields: SVec, + display_name: SString, ) -> FFIResult { unsafe { let inner = udwf.inner(); - let input_fields = rresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields)); + let input_fields = sresult_return!(rvec_wrapped_to_vec_fieldref(&input_fields)); - let field = rresult_return!(inner.field(WindowUDFFieldArgs::new( + let field = sresult_return!(inner.field(WindowUDFFieldArgs::new( &input_fields, display_name.as_str() ))); let schema = Arc::new(Schema::new(vec![field])); - RResult::ROk(WrappedSchema::from(schema)) + FFI_Result::Ok(WrappedSchema::from(schema)) } } unsafe extern "C" fn coerce_types_fn_wrapper( udwf: &FFI_WindowUDF, - arg_types: RVec, -) -> FFIResult> { + arg_types: SVec, +) -> FFIResult> { unsafe { let inner = udwf.inner(); - let arg_fields = rresult_return!(rvec_wrapped_to_vec_datatype(&arg_types)) + let arg_fields = sresult_return!(rvec_wrapped_to_vec_datatype(&arg_types)) .into_iter() .map(|dt| Field::new("f", dt, false)) .map(Arc::new) .collect::>(); - let return_fields = rresult_return!(fields_with_udf(&arg_fields, inner.as_ref())); + let return_fields = sresult_return!(fields_with_udf(&arg_fields, inner.as_ref())); let return_types = return_fields .into_iter() .map(|f| f.data_type().to_owned()) .collect::>(); - rresult!(vec_datatype_to_rvec_wrapped(&return_types)) + sresult!(vec_datatype_to_rvec_wrapped(&return_types)) } } @@ -362,7 +364,7 @@ impl WindowUDFImpl for ForeignWindowUDF { } fn sort_options(&self) -> Option { - let options: Option<&FFI_SortOptions> = self.udf.sort_options.as_ref().into(); + let options: Option<&FFI_SortOptions> = self.udf.sort_options.as_ref(); options.map(|s| s.into()) } @@ -372,7 +374,7 @@ impl WindowUDFImpl for ForeignWindowUDF { } #[repr(C)] -#[derive(Debug, StableAbi, Clone)] +#[derive(Debug, Clone)] pub struct FFI_SortOptions { pub descending: bool, pub nulls_first: bool, diff --git a/datafusion/ffi/src/udwf/partition_evaluator.rs b/datafusion/ffi/src/udwf/partition_evaluator.rs index 6820c6e335dd6..77e98f6870cac 100644 --- a/datafusion/ffi/src/udwf/partition_evaluator.rs +++ b/datafusion/ffi/src/udwf/partition_evaluator.rs @@ -19,8 +19,6 @@ use std::any::Any; use std::ffi::c_void; use std::ops::Range; -use abi_stable::StableAbi; -use abi_stable::std_types::{RResult, RVec}; use arrow::array::ArrayRef; use arrow::error::ArrowError; use datafusion_common::scalar::ScalarValue; @@ -29,33 +27,35 @@ use datafusion_expr::PartitionEvaluator; use datafusion_expr::window_state::WindowAggState; use prost::Message; +use stabby::vec::Vec as SVec; + use super::range::FFI_Range; use crate::arrow_wrappers::WrappedArray; -use crate::util::FFIResult; -use crate::{df_result, rresult, rresult_return}; +use crate::util::{FFI_Result, FFIResult}; +use crate::{df_result, sresult, sresult_return}; /// A stable struct for sharing [`PartitionEvaluator`] across FFI boundaries. /// For an explanation of each field, see the corresponding function /// defined in [`PartitionEvaluator`]. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_PartitionEvaluator { pub evaluate_all: unsafe extern "C" fn( evaluator: &mut Self, - values: RVec, + values: SVec, num_rows: usize, ) -> FFIResult, pub evaluate: unsafe extern "C" fn( evaluator: &mut Self, - values: RVec, + values: SVec, range: FFI_Range, - ) -> FFIResult>, + ) -> FFIResult>, pub evaluate_all_with_rank: unsafe extern "C" fn( evaluator: &Self, num_rows: usize, - ranks_in_partition: RVec, + ranks_in_partition: SVec, ) -> FFIResult, pub get_range: unsafe extern "C" fn( @@ -108,7 +108,7 @@ impl FFI_PartitionEvaluator { unsafe extern "C" fn evaluate_all_fn_wrapper( evaluator: &mut FFI_PartitionEvaluator, - values: RVec, + values: SVec, num_rows: usize, ) -> FFIResult { unsafe { @@ -118,7 +118,7 @@ unsafe extern "C" fn evaluate_all_fn_wrapper( .into_iter() .map(|v| v.try_into().map_err(DataFusionError::from)) .collect::>>(); - let values_arrays = rresult_return!(values_arrays); + let values_arrays = sresult_return!(values_arrays); let return_array = inner @@ -127,15 +127,15 @@ unsafe extern "C" fn evaluate_all_fn_wrapper( WrappedArray::try_from(&array).map_err(DataFusionError::from) }); - rresult!(return_array) + sresult!(return_array) } } unsafe extern "C" fn evaluate_fn_wrapper( evaluator: &mut FFI_PartitionEvaluator, - values: RVec, + values: SVec, range: FFI_Range, -) -> FFIResult> { +) -> FFIResult> { unsafe { let inner = evaluator.inner_mut(); @@ -143,23 +143,23 @@ unsafe extern "C" fn evaluate_fn_wrapper( .into_iter() .map(|v| v.try_into().map_err(DataFusionError::from)) .collect::>>(); - let values_arrays = rresult_return!(values_arrays); + let values_arrays = sresult_return!(values_arrays); // let return_array = (inner.evaluate(&values_arrays, &range.into())); // .and_then(|array| WrappedArray::try_from(&array).map_err(DataFusionError::from)); let scalar_result = - rresult_return!(inner.evaluate(&values_arrays, &range.into())); + sresult_return!(inner.evaluate(&values_arrays, &range.into())); let proto_result: datafusion_proto::protobuf::ScalarValue = - rresult_return!((&scalar_result).try_into()); + sresult_return!((&scalar_result).try_into()); - RResult::ROk(proto_result.encode_to_vec().into()) + FFI_Result::Ok(proto_result.encode_to_vec().into_iter().collect()) } } unsafe extern "C" fn evaluate_all_with_rank_fn_wrapper( evaluator: &FFI_PartitionEvaluator, num_rows: usize, - ranks_in_partition: RVec, + ranks_in_partition: SVec, ) -> FFIResult { unsafe { let inner = evaluator.inner(); @@ -175,7 +175,7 @@ unsafe extern "C" fn evaluate_all_with_rank_fn_wrapper( WrappedArray::try_from(&array).map_err(DataFusionError::from) }); - rresult!(return_array) + sresult!(return_array) } } @@ -188,7 +188,7 @@ unsafe extern "C" fn get_range_fn_wrapper( let inner = evaluator.inner(); let range = inner.get_range(idx, n_rows).map(FFI_Range::from); - rresult!(range) + sresult!(range) } } @@ -292,7 +292,7 @@ impl PartitionEvaluator for ForeignPartitionEvaluator { let values = values .iter() .map(WrappedArray::try_from) - .collect::, ArrowError>>()?; + .collect::, ArrowError>>()?; (self.evaluator.evaluate_all)(&mut self.evaluator, values, num_rows) }; @@ -310,7 +310,7 @@ impl PartitionEvaluator for ForeignPartitionEvaluator { let values = values .iter() .map(WrappedArray::try_from) - .collect::, ArrowError>>()?; + .collect::, ArrowError>>()?; let scalar_bytes = df_result!((self.evaluator.evaluate)( &mut self.evaluator, diff --git a/datafusion/ffi/src/udwf/partition_evaluator_args.rs b/datafusion/ffi/src/udwf/partition_evaluator_args.rs index ffad1f41ee42d..77b02b513419d 100644 --- a/datafusion/ffi/src/udwf/partition_evaluator_args.rs +++ b/datafusion/ffi/src/udwf/partition_evaluator_args.rs @@ -17,14 +17,13 @@ use std::sync::Arc; -use abi_stable::StableAbi; -use abi_stable::std_types::RVec; use arrow::error::ArrowError; use arrow::ffi::FFI_ArrowSchema; use arrow_schema::FieldRef; use datafusion_common::{DataFusionError, Result}; use datafusion_expr::function::PartitionEvaluatorArgs; use datafusion_physical_plan::PhysicalExpr; +use stabby::vec::Vec as SVec; use crate::arrow_wrappers::WrappedSchema; use crate::physical_expr::FFI_PhysicalExpr; @@ -34,10 +33,10 @@ use crate::util::rvec_wrapped_to_vec_fieldref; /// For an explanation of each field, see the corresponding function /// defined in [`PartitionEvaluatorArgs`]. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_PartitionEvaluatorArgs { - input_exprs: RVec, - input_fields: RVec, + input_exprs: SVec, + input_fields: SVec, is_reversed: bool, ignore_nulls: bool, } @@ -58,7 +57,8 @@ impl TryFrom> for FFI_PartitionEvaluatorArgs { .iter() .map(|input_type| FFI_ArrowSchema::try_from(input_type).map(WrappedSchema)) .collect::, ArrowError>>()? - .into(); + .into_iter() + .collect(); Ok(Self { input_exprs, diff --git a/datafusion/ffi/src/udwf/range.rs b/datafusion/ffi/src/udwf/range.rs index 19a908c5e2454..558fd058a67cf 100644 --- a/datafusion/ffi/src/udwf/range.rs +++ b/datafusion/ffi/src/udwf/range.rs @@ -17,13 +17,11 @@ use std::ops::Range; -use abi_stable::StableAbi; - /// A stable struct for sharing [`Range`] across FFI boundaries. /// For an explanation of each field, see the corresponding function /// defined in [`Range`]. #[repr(C)] -#[derive(Debug, StableAbi)] +#[derive(Debug)] pub struct FFI_Range { pub start: usize, pub end: usize, diff --git a/datafusion/ffi/src/util.rs b/datafusion/ffi/src/util.rs index db6eb0552d2aa..49ef46f49b002 100644 --- a/datafusion/ffi/src/util.rs +++ b/datafusion/ffi/src/util.rs @@ -17,59 +17,63 @@ use std::sync::Arc; -use abi_stable::std_types::{RResult, RString, RVec}; use arrow::datatypes::{DataType, Field}; use arrow::ffi::FFI_ArrowSchema; use arrow_schema::FieldRef; +use stabby::string::String as SString; +use stabby::vec::Vec as SVec; use crate::arrow_wrappers::WrappedSchema; +// Re-export for convenience +pub use crate::ffi_option::{FFI_Option, FFI_Result}; + /// Convenience type for results passed through the FFI boundary. Since the /// `DataFusionError` enum is complex and little value is gained from creating /// a FFI safe variant of it, we convert errors to strings when passing results -/// back. These are converted back and forth using the `df_result`, `rresult`, -/// and `rresult_return` macros. -pub type FFIResult = RResult; +/// back. These are converted back and forth using the `df_result`, `sresult`, +/// and `sresult_return` macros. +pub type FFIResult = FFI_Result; -/// This macro is a helpful conversion utility to convert from an abi_stable::RResult to a +/// This macro is a helpful conversion utility to convert from an FFIResult to a /// DataFusion result. #[macro_export] macro_rules! df_result { ( $x:expr ) => { - match $x { - abi_stable::std_types::RResult::ROk(v) => Ok(v), - abi_stable::std_types::RResult::RErr(err) => { + match Into::<::std::result::Result<_, _>>::into($x) { + Ok(v) => Ok(v), + Err(err) => { datafusion_common::ffi_err!("{err}") } } }; } -/// This macro is a helpful conversion utility to convert from a DataFusion Result to an abi_stable::RResult +/// This macro is a helpful conversion utility to convert from a DataFusion Result to an FFIResult. #[macro_export] -macro_rules! rresult { +macro_rules! sresult { ( $x:expr ) => { match $x { - Ok(v) => abi_stable::std_types::RResult::ROk(v), - Err(e) => abi_stable::std_types::RResult::RErr( - abi_stable::std_types::RString::from(e.to_string()), - ), + Ok(v) => $crate::ffi_option::FFI_Result::Ok(v), + Err(e) => $crate::ffi_option::FFI_Result::Err(stabby::string::String::from( + e.to_string().as_str(), + )), } }; } -/// This macro is a helpful conversion utility to convert from a DataFusion Result to an abi_stable::RResult -/// and to also call return when it is an error. Since you cannot use `?` on an RResult, this is designed +/// This macro is a helpful conversion utility to convert from a DataFusion Result to an FFIResult +/// and to also call return when it is an error. Since you cannot use `?` on an FFIResult, this is designed /// to mimic the pattern. #[macro_export] -macro_rules! rresult_return { +macro_rules! sresult_return { ( $x:expr ) => { match $x { Ok(v) => v, Err(e) => { - return abi_stable::std_types::RResult::RErr( - abi_stable::std_types::RString::from(e.to_string()), - ) + return $crate::ffi_option::FFI_Result::Err(stabby::string::String::from( + e.to_string().as_str(), + )) } } }; @@ -79,7 +83,7 @@ macro_rules! rresult_return { /// FFI friendly counterpart, [`WrappedSchema`] pub fn vec_fieldref_to_rvec_wrapped( fields: &[FieldRef], -) -> Result, arrow::error::ArrowError> { +) -> Result, arrow::error::ArrowError> { Ok(fields .iter() .map(FFI_ArrowSchema::try_from) @@ -92,7 +96,7 @@ pub fn vec_fieldref_to_rvec_wrapped( /// This is a utility function to convert an FFI friendly vector of [`WrappedSchema`] /// to their equivalent [`Field`]. pub fn rvec_wrapped_to_vec_fieldref( - fields: &RVec, + fields: &SVec, ) -> Result, arrow::error::ArrowError> { fields .iter() @@ -104,7 +108,7 @@ pub fn rvec_wrapped_to_vec_fieldref( /// FFI friendly counterpart, [`WrappedSchema`] pub fn vec_datatype_to_rvec_wrapped( data_types: &[DataType], -) -> Result, arrow::error::ArrowError> { +) -> Result, arrow::error::ArrowError> { Ok(data_types .iter() .map(FFI_ArrowSchema::try_from) @@ -117,7 +121,7 @@ pub fn vec_datatype_to_rvec_wrapped( /// This is a utility function to convert an FFI friendly vector of [`WrappedSchema`] /// to their equivalent [`DataType`]. pub fn rvec_wrapped_to_vec_datatype( - data_types: &RVec, + data_types: &SVec, ) -> Result, arrow::error::ArrowError> { data_types .iter() @@ -129,12 +133,13 @@ pub fn rvec_wrapped_to_vec_datatype( pub(crate) mod tests { use std::sync::Arc; - use abi_stable::std_types::{RResult, RString}; use datafusion::error::DataFusionError; use datafusion::prelude::SessionContext; use datafusion_execution::TaskContextProvider; + use stabby::string::String as SString; use crate::execution::FFI_TaskContextProvider; + use crate::ffi_option::FFI_Result; use crate::util::FFIResult; pub(crate) fn test_session_and_ctx() -> (Arc, FFI_TaskContextProvider) @@ -147,7 +152,7 @@ pub(crate) mod tests { } fn wrap_result(result: Result) -> FFIResult { - RResult::ROk(rresult_return!(result)) + FFI_Result::Ok(sresult_return!(result)) } #[test] @@ -155,14 +160,13 @@ pub(crate) mod tests { const VALID_VALUE: &str = "valid_value"; const ERROR_VALUE: &str = "error_value"; - let ok_r_result: FFIResult = - RResult::ROk(VALID_VALUE.to_string().into()); - let err_r_result: FFIResult = - RResult::RErr(ERROR_VALUE.to_string().into()); + let ok_r_result: FFIResult = FFI_Result::Ok(SString::from(VALID_VALUE)); + let err_r_result: FFIResult = + FFI_Result::Err(SString::from(ERROR_VALUE)); let returned_ok_result = df_result!(ok_r_result); assert!(returned_ok_result.is_ok()); - assert!(returned_ok_result.unwrap().to_string() == VALID_VALUE); + assert!(*returned_ok_result.unwrap() == *VALID_VALUE); let returned_err_result = df_result!(err_r_result); assert!(returned_err_result.is_err()); @@ -176,13 +180,16 @@ pub(crate) mod tests { datafusion_common::ffi_err!("{ERROR_VALUE}"); let returned_ok_r_result = wrap_result(ok_result); - assert!(returned_ok_r_result == RResult::ROk(VALID_VALUE.into())); + let std_result: Result = returned_ok_r_result.into(); + assert!(std_result == Ok(VALID_VALUE.into())); let returned_err_r_result = wrap_result(err_result); - assert!(returned_err_r_result.is_err()); + let std_result: Result = returned_err_r_result.into(); + assert!(std_result.is_err()); assert!( - returned_err_r_result + std_result .unwrap_err() + .as_str() .starts_with(format!("FFI error: {ERROR_VALUE}").as_str()) ); } diff --git a/datafusion/ffi/src/volatility.rs b/datafusion/ffi/src/volatility.rs index bc714ae59587d..48fce6f6cf117 100644 --- a/datafusion/ffi/src/volatility.rs +++ b/datafusion/ffi/src/volatility.rs @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -use abi_stable::StableAbi; use datafusion_expr::Volatility; -#[repr(C)] -#[derive(Debug, StableAbi, Clone)] +#[expect(non_camel_case_types)] +#[repr(u8)] +#[derive(Debug, Clone)] pub enum FFI_Volatility { Immutable, Stable, diff --git a/datafusion/ffi/tests/ffi_catalog.rs b/datafusion/ffi/tests/ffi_catalog.rs index 28bb5f406f53f..5a6547b8b0ebd 100644 --- a/datafusion/ffi/tests/ffi_catalog.rs +++ b/datafusion/ffi/tests/ffi_catalog.rs @@ -24,7 +24,6 @@ mod tests { use std::sync::Arc; use datafusion::catalog::{CatalogProvider, CatalogProviderList}; - use datafusion_common::DataFusionError; use datafusion_ffi::tests::utils::get_module; #[tokio::test] @@ -32,13 +31,7 @@ mod tests { let module = get_module()?; let (ctx, codec) = super::utils::ctx_and_codec(); - let ffi_catalog = - module - .create_catalog() - .ok_or(DataFusionError::NotImplemented( - "External catalog provider failed to implement create_catalog" - .to_string(), - ))?(codec); + let ffi_catalog = (module.create_catalog)(codec); let foreign_catalog: Arc = (&ffi_catalog).into(); let _ = ctx.register_catalog("fruit", foreign_catalog); @@ -59,13 +52,7 @@ mod tests { let module = get_module()?; let (ctx, codec) = super::utils::ctx_and_codec(); - let ffi_catalog_list = - module - .create_catalog_list() - .ok_or(DataFusionError::NotImplemented( - "External catalog provider failed to implement create_catalog_list" - .to_string(), - ))?(codec); + let ffi_catalog_list = (module.create_catalog_list)(codec); let foreign_catalog_list: Arc = (&ffi_catalog_list).into(); diff --git a/datafusion/ffi/tests/ffi_config.rs b/datafusion/ffi/tests/ffi_config.rs index ca0a3e31e8de6..4e5ad56722fe3 100644 --- a/datafusion/ffi/tests/ffi_config.rs +++ b/datafusion/ffi/tests/ffi_config.rs @@ -19,7 +19,7 @@ /// when the feature integration-tests is built #[cfg(feature = "integration-tests")] mod tests { - use datafusion::error::{DataFusionError, Result}; + use datafusion::error::Result; use datafusion_common::ScalarValue; use datafusion_common::config::{ConfigOptions, TableOptions}; use datafusion_execution::config::SessionConfig; @@ -31,13 +31,7 @@ mod tests { fn test_ffi_config_options_extension() -> Result<()> { let module = get_module()?; - let extension_options = - module - .create_extension_options() - .ok_or(DataFusionError::NotImplemented( - "External test library failed to implement create_extension_options" - .to_string(), - ))?(); + let extension_options = (module.create_extension_options)(); let mut config = ConfigOptions::new(); config.extensions.insert(extension_options); @@ -61,13 +55,7 @@ mod tests { fn test_ffi_table_options_extension() -> Result<()> { let module = get_module()?; - let extension_options = - module - .create_extension_options() - .ok_or(DataFusionError::NotImplemented( - "External test library failed to implement create_extension_options" - .to_string(), - ))?(); + let extension_options = (module.create_extension_options)(); let mut table_options = TableOptions::new(); table_options.extensions.insert(extension_options); @@ -92,13 +80,7 @@ mod tests { fn test_ffi_session_config_options_extension() -> Result<()> { let module = get_module()?; - let extension_options = - module - .create_extension_options() - .ok_or(DataFusionError::NotImplemented( - "External test library failed to implement create_extension_options" - .to_string(), - ))?(); + let extension_options = (module.create_extension_options)(); let mut config = SessionConfig::new().with_option_extension(extension_options); diff --git a/datafusion/ffi/tests/ffi_execution_plan.rs b/datafusion/ffi/tests/ffi_execution_plan.rs index d81f947dc80ed..a172c3d234570 100644 --- a/datafusion/ffi/tests/ffi_execution_plan.rs +++ b/datafusion/ffi/tests/ffi_execution_plan.rs @@ -49,12 +49,7 @@ mod tests { Arc::new(EmptyExec::new(schema)) } - let child_plan = - module - .create_empty_exec() - .ok_or(DataFusionError::NotImplemented( - "External module failed to implement create_empty_exec".to_string(), - ))?(); + let child_plan = (module.create_empty_exec)(); let child_plan: Arc = (&child_plan) .try_into() .expect("should be able create plan"); diff --git a/datafusion/ffi/tests/ffi_integration.rs b/datafusion/ffi/tests/ffi_integration.rs index 1be486589b722..4186bafc83d7a 100644 --- a/datafusion/ffi/tests/ffi_integration.rs +++ b/datafusion/ffi/tests/ffi_integration.rs @@ -26,7 +26,7 @@ mod tests { use arrow::datatypes::Schema; use datafusion::catalog::{TableProvider, TableProviderFactory}; - use datafusion::error::{DataFusionError, Result}; + use datafusion::error::Result; use datafusion_common::TableReference; use datafusion_common::ToDFSchema; use datafusion_expr::CreateExternalTable; @@ -42,11 +42,7 @@ mod tests { // By calling the code below, the table provided will be created within // the module's code. - let ffi_table_provider = table_provider_module.create_table().ok_or( - DataFusionError::NotImplemented( - "External table provider failed to implement create_table".to_string(), - ), - )?(synchronous, codec); + let ffi_table_provider = (table_provider_module.create_table)(synchronous, codec); // In order to access the table provider within this executable, we need to // turn it into a `TableProvider`. @@ -80,11 +76,8 @@ mod tests { let table_provider_module = get_module()?; let (ctx, codec) = super::utils::ctx_and_codec(); - let ffi_table_provider_factory = table_provider_module - .create_table_factory() - .ok_or(DataFusionError::NotImplemented( - "External table provider factory failed to implement create".to_string(), - ))?(codec); + let ffi_table_provider_factory = + (table_provider_module.create_table_factory)(codec); let foreign_table_provider_factory: Arc = (&ffi_table_provider_factory).into(); diff --git a/datafusion/ffi/tests/ffi_udaf.rs b/datafusion/ffi/tests/ffi_udaf.rs index f219979a85062..7df3404d7421b 100644 --- a/datafusion/ffi/tests/ffi_udaf.rs +++ b/datafusion/ffi/tests/ffi_udaf.rs @@ -23,7 +23,7 @@ mod tests { use arrow::array::Float64Array; use datafusion::common::record_batch; - use datafusion::error::{DataFusionError, Result}; + use datafusion::error::Result; use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl}; use datafusion::prelude::{SessionContext, col}; use datafusion_catalog::MemTable; @@ -34,12 +34,7 @@ mod tests { async fn test_ffi_udaf() -> Result<()> { let module = get_module()?; - let ffi_sum_func = - module - .create_sum_udaf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_udaf".to_string(), - ))?(); + let ffi_sum_func = (module.create_sum_udaf)(); let foreign_sum_func: Arc = (&ffi_sum_func).into(); let udaf = AggregateUDF::new_from_shared_impl(foreign_sum_func); @@ -76,12 +71,7 @@ mod tests { async fn test_ffi_grouping_udaf() -> Result<()> { let module = get_module()?; - let ffi_stddev_func = - module - .create_stddev_udaf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_udaf".to_string(), - ))?(); + let ffi_stddev_func = (module.create_stddev_udaf)(); let foreign_stddev_func: Arc = (&ffi_stddev_func).into(); let udaf = AggregateUDF::new_from_shared_impl(foreign_stddev_func); @@ -137,25 +127,14 @@ mod tests { async fn udf_as_input_to_udf() -> Result<()> { let module = get_module()?; - let ffi_abs_func = - module - .create_scalar_udf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_scalar_udf" - .to_string(), - ))?(); + let ffi_abs_func = (module.create_scalar_udf)(); let foreign_abs_func: Arc = (&ffi_abs_func).into(); let abs_udf = ScalarUDF::new_from_shared_impl(foreign_abs_func); let ctx = SessionContext::new(); ctx.deregister_udf("abs"); - let ffi_sum_func = - module - .create_sum_udaf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_udaf".to_string(), - ))?(); + let ffi_sum_func = (module.create_sum_udaf)(); let foreign_sum_func: Arc = (&ffi_sum_func).into(); let udaf = AggregateUDF::new_from_shared_impl(foreign_sum_func); diff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rs index 02dfba599f316..6e6cb31f53133 100644 --- a/datafusion/ffi/tests/ffi_udf.rs +++ b/datafusion/ffi/tests/ffi_udf.rs @@ -22,7 +22,7 @@ mod tests { use arrow::array::{Array, AsArray}; use arrow::datatypes::DataType; use datafusion::common::record_batch; - use datafusion::error::{DataFusionError, Result}; + use datafusion::error::Result; use datafusion::logical_expr::{ScalarUDF, ScalarUDFImpl}; use datafusion::prelude::{SessionContext, col}; use datafusion_execution::config::SessionConfig; @@ -38,13 +38,7 @@ mod tests { async fn test_scalar_udf() -> Result<()> { let module = get_module()?; - let ffi_abs_func = - module - .create_scalar_udf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_scalar_udf" - .to_string(), - ))?(); + let ffi_abs_func = (module.create_scalar_udf)(); let foreign_abs_func: Arc = (&ffi_abs_func).into(); let udf = ScalarUDF::new_from_shared_impl(foreign_abs_func); @@ -76,13 +70,7 @@ mod tests { async fn test_nullary_scalar_udf() -> Result<()> { let module = get_module()?; - let ffi_abs_func = - module - .create_nullary_udf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_scalar_udf" - .to_string(), - ))?(); + let ffi_abs_func = (module.create_nullary_udf)(); let foreign_abs_func: Arc = (&ffi_abs_func).into(); let udf = ScalarUDF::new_from_shared_impl(foreign_abs_func); @@ -107,12 +95,7 @@ mod tests { async fn test_config_on_scalar_udf() -> Result<()> { let module = get_module()?; - let ffi_udf = - module - .create_timezone_udf() - .ok_or(DataFusionError::NotImplemented( - "External module failed to implement create_timezone_udf".to_string(), - ))?(); + let ffi_udf = (module.create_timezone_udf)(); let foreign_udf: Arc = (&ffi_udf).into(); let udf = ScalarUDF::new_from_shared_impl(foreign_udf); diff --git a/datafusion/ffi/tests/ffi_udtf.rs b/datafusion/ffi/tests/ffi_udtf.rs index ab7818932959c..69e5de90e1364 100644 --- a/datafusion/ffi/tests/ffi_udtf.rs +++ b/datafusion/ffi/tests/ffi_udtf.rs @@ -26,7 +26,7 @@ mod tests { use arrow::array::{ArrayRef, create_array}; use datafusion::catalog::TableFunctionImpl; - use datafusion::error::{DataFusionError, Result}; + use datafusion::error::Result; use datafusion_ffi::tests::utils::get_module; /// This test validates that we can load an external module and use a scalar @@ -37,12 +37,7 @@ mod tests { let module = get_module()?; let (ctx, codec) = super::utils::ctx_and_codec(); - let ffi_table_func = module - .create_table_function() - .ok_or(DataFusionError::NotImplemented( - "External table function provider failed to implement create_table_function" - .to_string(), - ))?(codec); + let ffi_table_func = (module.create_table_function)(codec); let foreign_table_func: Arc = ffi_table_func.into(); ctx.register_udtf("my_range", foreign_table_func); diff --git a/datafusion/ffi/tests/ffi_udwf.rs b/datafusion/ffi/tests/ffi_udwf.rs index c4e889b796008..66f2621d5fe63 100644 --- a/datafusion/ffi/tests/ffi_udwf.rs +++ b/datafusion/ffi/tests/ffi_udwf.rs @@ -22,7 +22,7 @@ mod tests { use std::sync::Arc; use arrow::array::{ArrayRef, create_array}; - use datafusion::error::{DataFusionError, Result}; + use datafusion::error::Result; use datafusion::logical_expr::expr::Sort; use datafusion::logical_expr::{ExprFunctionExt, WindowUDF, WindowUDFImpl, col}; use datafusion::prelude::SessionContext; @@ -33,13 +33,7 @@ mod tests { async fn test_rank_udwf() -> Result<()> { let module = get_module()?; - let ffi_rank_func = - module - .create_rank_udwf() - .ok_or(DataFusionError::NotImplemented( - "External table provider failed to implement create_scalar_udf" - .to_string(), - ))?(); + let ffi_rank_func = (module.create_rank_udwf)(); let foreign_rank_func: Arc = (&ffi_rank_func).into(); let udwf = WindowUDF::new_from_shared_impl(foreign_rank_func);