From 3e3bcb1b7dd100c87d91aef3c77c05270e91aa12 Mon Sep 17 00:00:00 2001 From: Junru Shao Date: Mon, 12 Jan 2026 14:14:22 -0800 Subject: [PATCH 1/2] doc: abi overview --- docs/concepts/abi_overview.md | 462 ----------------------- docs/concepts/abi_overview.rst | 527 +++++++++++++++++++++++++++ docs/concepts/any.rst | 42 +-- docs/concepts/func_module.rst | 221 +++-------- docs/concepts/object_and_class.rst | 112 ++---- docs/concepts/tensor.rst | 150 +------- docs/guides/compiler_integration.md | 2 +- docs/index.rst | 2 +- examples/abi_overview/example_code.c | 267 ++++++++++++++ include/tvm/ffi/c_api.h | 14 + 10 files changed, 924 insertions(+), 875 deletions(-) delete mode 100644 docs/concepts/abi_overview.md create mode 100644 docs/concepts/abi_overview.rst create mode 100644 examples/abi_overview/example_code.c diff --git a/docs/concepts/abi_overview.md b/docs/concepts/abi_overview.md deleted file mode 100644 index 573f7e40..00000000 --- a/docs/concepts/abi_overview.md +++ /dev/null @@ -1,462 +0,0 @@ - - - - - - - - - - - - - - - - - -# ABI Overview - -This section provides an overview of the ABI convention of TVM FFI. The ABI -is designed around the following key principles: - -- **Stable C ABI:** Core ABI is defined on top of a stable C ABI. -- **Minimal and efficient:** Keep things simple when possible and bring close-to-metal efficiency. -- **Focus on machine learning systems:** while also ensuring reasonable extensibility. - -To explain the concepts in the following sections, we will write in **low-level C/C++ code** when possible, -so the code itself illustrates the low-level semantics of how to work with the ABI convention. -These can serve as references for how to build language bindings and compiler codegen for the ABI. - -```{note} -The authoritative ABI specifications are defined in [tvm/ffi/c_api.h](https://github.com/apache/tvm-ffi/blob/main/include/tvm/ffi/c_api.h) for core ABI, -and [tvm/ffi/extra/c_env_api.h](https://github.com/apache/tvm-ffi/blob/main/include/tvm/ffi/extra/c_env_api.h) for extra support features -such as stream handling. This document provides explanations about design concepts and rationales. -``` - -## Simplified Example - -Before diving into details, it is helpful to review at a high level -what happens when a function is called in TVM FFI ABI. -One main design goal here is to represent all kinds of functions in a single -unified C signature. Please review the following -simplified code example that illustrates the key idea: - -```c++ -// simplified struct for TVMFFIAny -typedef struct TVMFFIAny { - int32_t type_index; - uint32_t zero_padding; - // union values - union { - int64_t v_int64; // integers - double v_float64; // floating-point numbers - const char* v_c_str; // raw C-string - }; -}; - -// This is the signature of TVM FFI function ABI -typedef int (*TVMFFISafeCallType)( - void* handle, const TVMFFIAny* args, int32_t num_args, TVMFFIAny* result -); - -// An example function signature -int MyFunc(const char* param0, int param1); - -// This is what MyFunc looks like when exposed through TVM FFI ABI -int MyFuncTVMFFISafeCall( - void* handle, const TVMFFIAny* args, int32_t num_args, TVMFFIAny* result -) { - assert(args[0].type_index == kTVMFFIRawStr); - assert(args[1].type_index == kTVMFFInt); - result->type_index = kTVMFFInt; - result->v_int64 = MyFunc(args[0].v_c_str, args[1].v_int64); - // return value indicates no error occurred - return 0; -} - -// This is how we call the MyFuncTVMFFISafeCall -// this can happen on the caller side in another language (e.g. python) -int CallTVMFFISafeCall(const char* param0, int param1) { - // arguments on stack - TVMFFIAny args[2], result; - args[0].type_index = kTVMFFIRawStr; - args[0].v_c_str = param0; - args[1].type_index = kTVMFFInt; - args[1].v_int64 = param1; - result.type_index = kTVMFFINone; - // In this case we do not need handle - // handle is used to hold closure pointers - void* handle = nullptr; - int num_args = 2; - MyFuncTVMFFISafeCall(handle, args, num_args, &result); - return result.v_int64; -} -``` - -At a high level, the `TVMFFISafeCallType` signature does the following things: - -- Arguments and return values are stored in structured `TVMFFIAny` - - Each value comes with a `type_index` to indicate its type - - Values are stored in union fields, depending on the specific type. -- Caller can explicitly store the type index and value into - a stack of `TVMFFIAny`. -- Callee can load the parameters from args and check their type indices. - -In this way, the same `TVMFFISafeCallType` can be used to represent any function -that contains an arbitrary number of arguments and types that can be identified by `type_index`. -Of course, this is a simplified example and we did not touch on specific details -like Any value format and error handling. The following sections will provide a more systematic -treatment of each of these specific topics. -You can keep this example in mind as the overall picture and refine it as you read through -the following sections. - -## TVMFFIAny Storage Format - -To start with, we need a mechanism to store the values that are passed across machine learning frameworks. -It achieves this using a core data structure called TVMFFIAny. - -```c++ -typedef struct TVMFFIAny { - int32_t type_index; - union { // 4 bytes - uint32_t zero_padding; - uint32_t small_str_len; - }; - // union values - union { - int64_t v_int64; // integers - double v_float64; // floating-point numbers - void* v_ptr; // typeless pointers - const char* v_c_str; // raw C-string - TVMFFIObject* v_obj; // ref counted objects - DLDataType v_dtype; // data type - DLDevice v_device; // device - char v_bytes[8]; // small string - ... - }; -} TVMFFIAny; -``` - -TVMFFIAny is a 16-byte C structure that follows the design principle of tagged-union: - -- `type_index` helps us identify the type being stored. -- The value union part is designed to store the value: - - Small POD values (like integers and floats) are stored directly as "on-stack" values. - - `v_obj` can also point to a managed heap-allocated object, which we will discuss next. -- The second field stores metadata for small strings. - -### Storing a POD Value - -There are many values that are plain-old-data types. In such cases, we store them directly -on-stack in the value part of the TVMFFIAny. The following example shows how to store -an int. - -```c++ -void SetIntValue(TVMFFIAny* any, int value) { - // must zero the entire space first - any->type_index = kTVMFFIInt; - any->zero_padding = 0; - any->v_int64 = value; -} -``` - -:::{note} - -We **must zero the content that is not being used** by -the current value type. The following example shows a common place -where mistakes can be made when we forget to zero the value field -on 32-bit platforms (where pointers only fill the 32-bit part of the value). - -```c++ -void SetOpaquePtrValue(TVMFFIAny* any, void* opaque_ptr) { - any->type_index = kTVMFFIOpaquePtr; - // must zero the padding - any->zero_padding = 0; - // the zeroing is needed for 32-bit platforms! - any->v_uint64 = 0; - any->v_ptr = opaque_ptr; -} -``` - -**Rationale:** Such invariants allow us to directly compare -and hash TVMFFIAny in bytes for quick equality checks without going through -type index switching. -::: - -(object-storage-format)= - -## Object Storage Format - -When TVMFFIAny points to a heap-allocated object (such as n-dimensional arrays), -we adopt a unified object storage format, defined as follows: - -```c++ -typedef struct TVMFFIObject { - uint64_t combined_ref_count; - int32_t type_index; - uint32_t __padding; - union { - void (*deleter)(struct TVMFFIObject* self, int flags); - int64_t __ensure_align; - }; -} TVMFFIObject; -``` - -`TVMFFIObject` defines a common 24-byte intrusive header that all in-memory objects share: - -- `combined_ref_count` packs strong and weak reference counter of the object into a single 64bit field - - The lower 32bits stores the strong atomic reference counter: - `strong_ref_count = combined_ref_count & 0xFFFFFFFF` - - The higher 32bits stores the weak atomic reference counter: - `weak_ref_count = (combined_ref_count >> 32) & 0xFFFFFFFF` -- `type_index` helps us identify the type being stored, which is consistent with `TVMFFIAny.type_index`. -- `deleter` should be called when either the strong or weak ref counter goes to zero. - - The flags are set to indicate the event of either weak or strong going to zero, or both. - - When strong reference counter gets to zero, the deleter needs to call the destructor of the object. - - When weak reference counter gets to zero, the deleter needs to free the memory allocated by self. - -**Rationales:** There are several considerations when designing the data structure: - -- `type_index` enables runtime dynamic type checking and casting. -- We introduce weak/strong ref counters so we can be compatible with systems that need weak pointers. -- The weak ref counter is kept as 32-bit so we can pack the object header as 24 bytes. -- `deleter` ensures that objects allocated from one language/runtime can be safely deleted in another. - -The object format provides a unified way to manage object life-cycle and dynamic type casting -for heap-allocated objects, including Shape, Tensor, -Function, Array, Map and other custom objects. - -### DLPack Compatible Tensor - -We provide first-class support for DLPack raw unmanaged pointer support as well as a managed Tensor object that -directly adopts the DLPack DLTensor layout. The overall layout of the Tensor object is as follows: - -```c++ -struct TensorObj: public ffi::Object, public DLTensor { -}; -``` - -That means we can read out the array buffer information from an `TVMFFIAny` -in the following way: - -```c++ -DLTensor* ReadDLTensorPtr(const TVMFFIAny *value) { - if (value->type_index == kTVMFFIDLTensorPtr) { - return static_cast(value->v_ptr); - } - assert(value->type_index == kTVMFFITensor); - return reinterpret_cast( - reinterpret_cast(value->v_obj) + sizeof(TVMFFIObject)); -} -``` - -The above code can be used as a reference to implement compiler codegen for data. -Note that the C++ API automatically handles such conversion. - -### Advanced: Dynamic Type Index - -The `TVMFFITypeIndex` defines a set of type indices. Each built-in type has a corresponding statically -assigned type index that is defined in the enum. Static type indices should be sufficient for most -library use cases. -For advanced use cases we also support user-defined objects whose `type_index` are assigned at startup time -by calling `TVMFFITypeGetOrAllocIndex` with a unique -`type_key` string. This design allows us to enable decentralized extension of the objects as long as the `type_key` -values are unique by appending namespace prefix to the key. - -## AnyView and Managed Any - -```{seealso} -For a comprehensive tutorial on Any including ownership semantics, extraction methods, -and common patterns, see {doc}`any`. -``` - -An `TVMFFIAny` can either be treated as a strongly managed value (corresponding to `ffi::Any` in C++), -or an unmanaged value (corresponding to `ffi::AnyView` in C++). - -- For POD types, there is no difference between the two. -- For object types, copying of AnyView should not change reference counters, while copying and deletion - of managed Any should result in increase and decrease of strong reference counters. -- When we convert AnyView to Any, we will convert raw C string `const char*` and `const TVMFFIByteArray*` - into their managed counterparts (String and Bytes). -- C API function `TVMFFIAnyViewToOwnedAny` is provided to perform such conversion. - -Unless the user is writing a compiler backend that needs low-level C style access, we encourage use of the -C++ API to automatically manage conversion and casting between normal types and Any. The following code -shows some example usage of the C++ API. - -```c++ -#include - -void AnyExample() { - namespace ffi = tvm::ffi; - // Here is a managed any - ffi::Any value = "hello world"; - // explicit cast to a specific type - ffi::String str_value = value.cast(); - // copy int to value - value = 1; - // copy into a view - ffi::AnyView view = value; - // cast view back to int - std::cout << "Value is " << view.cast() << std::endl; -} -``` - -`ffi::Any` can serve as a container type to hold managed values that can be recognized by the TVM FFI system. -They can be composed with container structures such as `Map`, `Array` to represent various -broad patterns in APIs that may appear in ML systems. - -## Function Calling Convention - -As discussed in the overview, we need to consider foreign function calls as first-class citizens. We adopt a single standard C function as follows: - -```c++ -typedef int (*TVMFFISafeCallType)( - void* handle, const TVMFFIAny* args, int32_t num_args, TVMFFIAny* result -); -``` - -The handle contains the pointer to the function object itself, allowing us to support closures. args and num_args describe the input arguments and results store the return value. When args and results contain heap-managed objects, we expect the caller to own args and result. - -```{note} -Before calling the function, caller must set `result->type_index` to be kTVMFFINone, or any type index that do not corresponds -to an on-heap object. - -**Rationale:** Simplifies callee implementation as initial state of result can be viewed as managed Any. -``` - -We call this approach a packed function, as it provides a single signature to represent all functions in a "type-erased" way. It saves the need to declare and jit shim for each FFI function call while maintaining reasonable efficiency. This mechanism enables the following scenarios: - -- Calling from Dynamic Languages (e.g., Python): we provide a tvm_ffi binding that prepares the args based on dynamically examining Python arguments passed in. -- Calling from Static Languages (e.g., C++): For static languages, we can leverage C++ templates to directly instantiate the arguments on the stack, saving the need for dynamic examination. -- Dynamic language Callbacks: the signature enables us to easily bring dynamic language (Python) callbacks as ffi::Function, as we can take each argument and convert to the dynamic values. -- Efficiency: In practice, we find this approach is sufficient for machine learning focused workloads. For example, we can get to microsecond level overhead for Python/C++ calls, which is generally similar to overhead for eager mode. When both sides of calls are static languages, the overhead will go down to tens of nanoseconds. As a side note, although we did not find it necessary, the signature still leaves room for link time optimization (LTO), when both sides are static languages with a known symbol and linked into a single binary when we inline the callee into caller side and the stack argument memory passing into register passing. - -We support first-class Function objects that allow us to also pass function/closures from different places around, enabling cool usages such as quick python callback for prototyping, and dynamic Functor creation for driver-based kernel launching. - -## Error Handling - -Most TVM FFI C API calls, including `TVMFFISafeCallType` uses the return value to -indicate whether an error happens. When an error happens during a function call, -a non-zero value will be returned. The callee needs also to set the error through `TVMFFIErrorSetRaisedFromCStr` or `TVMFFIErrorSetRaised` API, which stores -the error on a thread-local storage. - -```c++ -// Example function that raises an error -int ErrorFunc(void* handle, const TVMFFIAny* args, int num_args, TVMFFIAny *result) { - const char* error_kind = "RuntimeError"; - const char* error_msg = "error message"; - // set the thread-local error state - TVMFFIErrorSetRaisedFromCStr(error_kind, error_msg); - return -1; -} -``` - -The caller can retrieve the error from thread-local error storage -using `TVMFFIErrorMoveFromRaised` function. -The ABI stores Error also as a specific Object, -the overall error object is stored as follows - -```c++ -/*! - * \brief Error cell used in error object following header. - */ -typedef struct { - /*! \brief The kind of the error. */ - TVMFFIByteArray kind; - /*! \brief The message of the error. */ - TVMFFIByteArray message; - /*! - * \brief The backtrace of the error. - * - * The backtrace is in the order of recent call first from the top of the stack - * to the bottom of the stack. This order makes it helpful for appending - * the extra backtrace to the end as we go up when error is propagated. - * - * When printing out, we encourage reverse the order of lines to make it - * align with python style. - */ - TVMFFIByteArray backtrace; - /*! - * \brief Function handle to update the backtrace of the error. - * \param self The self object handle. - * \param backtrace The backtrace to update. - * \param update_mode The mode to update the backtrace, - * can be either kTVMFFIBacktraceUpdateModeReplace, kTVMFFIBacktraceUpdateModeAppend. - */ - void (*update_backtrace)( - TVMFFIObjectHandle self, const TVMFFIByteArray* backtrace, int32_t update_mode); -} TVMFFIErrorCell; - -// error object -class ErrorObj : public ffi::Object, public TVMFFIErrorCell { -}; -``` - -The error object stores kind, message and backtrace as string. When possible, -we store the backtrace in the same format of python-style (see an example as follows): - -```text -File "src/extension.cc", line 45, in void my_ffi_extension::RaiseError(tvm::ffi::String) -``` - -We provide C++ object `ffi::Error` that can be throwed as exception in c++ environment. When we encounter -the C ABI boundary, we will catch the error and call `TVMFFIErrorSetRaised` to propagate the error -to the caller safely. -`TVMFFIErrorSetRaisedFromCStr` is a convenient method to set error directly from C string and can be useful in compiler -backend construction to implement features such as assert. -We also provide `TVMFFIErrorSetRaisedFromCStrParts` to concat reusable parts in the error message. - -**Rationales:** The error object contains minimal but sufficient information to reconstruct structured -error in python side. We opt-for thread-local error state as it simplifies overall support. - -## String and Bytes - -The ABI supports strings and bytes as first-class citizens. A string can take multiple forms that are identified by -its `type_index`. - -- `kTVMFFIRawStr`: raw C string terminated by `\0`. -- `kTVMFFISmallStr`: small string, the length is stored in `small_str_len` and data is stored in `v_bytes`. -- `kTVMFFIStr`: on-heap string object for strings that are longer than 7 characters. - -The following code shows the layout of the on-heap string object. - -```c++ -// span-like data structure to store header and length -typedef struct { - const char* data; - size_t size; -} TVMFFIByteArray; - -// showcase the layout of the on-heap string. -class StringObj : public ffi::Object, public TVMFFIByteArray { -}; -``` - -The following code shows how to read a string from `TVMFFIAny` - -```c++ -TVMFFIByteArray ReadString(const TVMFFIAny *value) { - TVMFFIByteArray ret; - if (value->type_index == kTVMFFIRawStr) { - ret.data = value->v_c_str; - ret.size = strlen(ret.data); - } else if (value->type_index == kTVMFFISmallStr) { - ret.data = value->v_bytes; - ret.size = value->small_str_len; - } else { - assert(value->type_index == kTVMFFIStr); - ret = *reinterpret_cast( - reinterpret_cast(value->v_obj) + sizeof(TVMFFIObject)); - } - return ret; -} -``` - -Similarly, we have type indices to represent bytes. The C++ API provides classes -`ffi::String` and `ffi::Bytes` to enable the automatic conversion of these values with Any storage format. - -**Rationales:** Separate string and bytes enable clear mappings from the Python side. Small string allows us to -store short names on-stack. To favor 8-byte alignment (v_bytes) and keep things simple, we did not further -pack characters into the `small_len` field. diff --git a/docs/concepts/abi_overview.rst b/docs/concepts/abi_overview.rst new file mode 100644 index 00000000..56291ebf --- /dev/null +++ b/docs/concepts/abi_overview.rst @@ -0,0 +1,527 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +ABI Overview +============ + +.. hint:: + + Authoritative ABI specifications are defined in + + - C header `tvm/ffi/c_api.h `_, which contains the core ABI, and + - C header `tvm/ffi/extra/c_env_api.h `_, which contains extra support features. + +The TVM-FFI ABI is designed around the following key principles: + +- **Minimal and efficient.** Keep things simple and deliver close-to-metal performance. +- **Stability guarantee.** The ABI remains stable across compiler versions and is independent of host languages or frameworks. +- **Expressive for machine learning.** Native support for tensors, shapes, and data types commonly used in ML workloads. +- **Extensible.** The ABI supports user-defined types and features through a dynamic type registration system. + +This tutorial covers common concepts and usage patterns of the TVM-FFI ABI, with low-level C code examples for precise reference. + +.. important:: + C code is used for clarity, precision and friendliness to compiler builders. + And C code can be readily translated into code generators such as LLVM IR builder. + +Any and AnyView +--------------- + +.. seealso:: + + :doc:`any` for :cpp:class:`~tvm::ffi::Any` and :cpp:class:`~tvm::ffi::AnyView` usage patterns. + +At the core of TVM-FFI is :cpp:class:`TVMFFIAny`, a 16-byte tagged union that can hold any value +recognized by the FFI system. It enables type-erased value passing across language boundaries. + +.. dropdown:: C ABI Reference: :cpp:class:`TVMFFIAny` + :icon: code + + .. literalinclude:: ../../include/tvm/ffi/c_api.h + :language: c + :start-after: [TVMFFIAny.begin] + :end-before: [TVMFFIAny.end] + :caption: tvm/ffi/c_api.h + +**Ownership.** :cpp:class:`TVMFFIAny` struct can represent either an owning or a borrowing reference. +These two ownership patterns are formalized by the C++ wrapper classes :cpp:class:`~tvm::ffi::Any` and :cpp:class:`~tvm::ffi::AnyView`, +which have identical memory layouts but different :ref:`ownership semantics `: + +- **Owning:** :cpp:class:`tvm::ffi::Any` - reference-counted, manages object lifetime +- **Borrowing:** :cpp:class:`tvm::ffi::AnyView` - non-owning view, caller must ensure validity + +.. note:: + To convert a borrowing :cpp:class:`~tvm::ffi::AnyView` to an owning :cpp:class:`~tvm::ffi::Any`, use :cpp:func:`TVMFFIAnyViewToOwnedAny`. + +**Runtime Type Index.** The ``type_index`` field identifies what kind of value is stored: + +- :ref:`Atomic POD types ` (``type_index`` < :cpp:enumerator:`kTVMFFIStaticObjectBegin `): + Stored inline in the payload union without heap allocation or reference counting. +- :ref:`Object types ` (``type_index`` >= :cpp:enumerator:`kTVMFFIStaticObjectBegin `): + Stored as pointers to heap-allocated, reference-counted TVM-FFI objects. + +.. important:: + The TVM-FFI type index system does not rely on C++ RTTI. + + +Construct Any +~~~~~~~~~~~~~ + +**From atomic POD types.** The following C code constructs a :cpp:class:`TVMFFIAny` from an integer: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Any_AnyView.FromInt_Float.begin] + :end-before: [Any_AnyView.FromInt_Float.end] + +Set the ``type_index`` from :cpp:enum:`TVMFFITypeIndex` and assign the corresponding payload field. + +.. important:: + + Always zero the ``zero_padding`` field and any unused bytes in the value union. + This invariant enables direct byte comparison and hashing of :cpp:class:`TVMFFIAny` values. + +**From object types.** The following C code constructs a :cpp:class:`TVMFFIAny` from a heap-allocated object: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Any_AnyView.FromObjectPtr.begin] + :end-before: [Any_AnyView.FromObjectPtr.end] + +When ``IS_OWNING_ANY`` is ``true`` (owning :cpp:class:`~tvm::ffi::Any`), this increments the object's reference count. + +.. _abi-destruct-any: + +Destruct Any +~~~~~~~~~~~~ + +The following C code destroys a :cpp:class:`TVMFFIAny`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Any_AnyView.Destroy.begin] + :end-before: [Any_AnyView.Destroy.end] + +When ``IS_OWNING_ANY`` is ``true`` (owning :cpp:class:`~tvm::ffi::Any`), this decrements the object's reference count. + +Extract from Any +~~~~~~~~~~~~~~~~ + +**Extract an atomic POD.** The following C code extracts an integer or float from a :cpp:class:`TVMFFIAny`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Any_AnyView.GetInt_Float.begin] + :end-before: [Any_AnyView.GetInt_Float.end] + +Implicit type conversion may occur. For example, when extracting a float from a :cpp:class:`TVMFFIAny` +that holds an integer, the integer is cast to a float. + +**Extract a DLTensor.** A :c:struct:`DLTensor` may originate from either a raw pointer or a heap-allocated :cpp:class:`~tvm::ffi::TensorObj`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Any_AnyView.GetDLTensor.begin] + :end-before: [Any_AnyView.GetDLTensor.end] + +**Extract a TVM-FFI object.** TVM-FFI objects are always heap-allocated and reference-counted, +with ``type_index`` >= :cpp:enumerator:`kTVMFFIStaticObjectBegin `: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Any_AnyView.GetObject.begin] + :end-before: [Any_AnyView.GetObject.end] + +To take ownership of the returned value, increment the reference count via :cpp:func:`TVMFFIObjectIncRef`. +Release ownership later via :cpp:func:`TVMFFIObjectDecRef`. + +.. _abi-object: + +Object +------ + +.. seealso:: + + :doc:`object_and_class` for the object system and reflection. + +TVM-FFI Object (:cpp:class:`TVMFFIObject`) is the cornerstone of TVM-FFI's stable yet extensible type system. + +.. dropdown:: C ABI Reference: :cpp:class:`TVMFFIObject` + :icon: code + + .. literalinclude:: ../../include/tvm/ffi/c_api.h + :language: c + :start-after: [TVMFFIObject.begin] + :end-before: [TVMFFIObject.end] + :caption: tvm/ffi/c_api.h + +All TVM-FFI objects share these characteristics: + +- Heap-allocated and reference-counted +- Layout-stable 24-byte header containing reference counts, type index, and deleter callback +- Type index >= :cpp:enumerator:`kTVMFFIStaticObjectBegin ` + +**Dynamic Type System.** Classes can be registered at runtime via :cpp:func:`TVMFFITypeGetOrAllocIndex`, +with support for single inheritance. See :ref:`type-checking-and-casting` for usage details. + +A small **static section** between :cpp:enumerator:`kTVMFFIStaticObjectBegin ` +and :cpp:enumerator:`kTVMFFIDynObjectBegin ` +is reserved for static object types, for example, + +- Strings (:cpp:enumerator:`kTVMFFIStr `) and Bytes (:cpp:enumerator:`kTVMFFIBytes `): Section :ref:`abi-string-and-byte` +- Errors (:cpp:enumerator:`kTVMFFIError `): Section :ref:`abi-exception`. +- Functions (:cpp:enumerator:`kTVMFFIFunction `): Section :ref:`abi-function`. +- Tensors (:cpp:enumerator:`kTVMFFITensor `): Section :ref:`abi-tensor`. +- Miscellaneous: + Modules (:cpp:enumerator:`kTVMFFIModule `), + Arrays (:cpp:enumerator:`kTVMFFIArray `), + Maps (:cpp:enumerator:`kTVMFFIMap `), + Shapes (:cpp:enumerator:`kTVMFFIShape `), + Opaque Python objects (:cpp:enumerator:`kTVMFFIOpaquePyObject `). + +.. _abi-object-ownership: + +Ownership Management +~~~~~~~~~~~~~~~~~~~~ + +Ownership is managed via reference counting, which includes both strong and weak references. +Two C APIs manage strong reference counting: + +- :cpp:func:`TVMFFIObjectIncRef`: Acquire strong ownership by incrementing the reference count +- :cpp:func:`TVMFFIObjectDecRef`: Release strong ownership by decrementing the reference count + +The ``deleter`` callback (:cpp:member:`TVMFFIObject::deleter`) executes when the strong or weak count reaches zero with different flags. +See :ref:`object-reference-counting` for details. + +**Move ownership from Any/AnyView.** The following C code transfers ownership from an owning :cpp:class:`~tvm::ffi::Any` to an object pointer: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Object.MoveFromAny.begin] + :end-before: [Object.MoveFromAny.end] + +Since :cpp:class:`~tvm::ffi::AnyView` is non-owning (``IS_OWNING_ANY`` is ``false``), +acquiring ownership requires explicitly incrementing the reference count. + +**Release ownership.** The following C code releases ownership of a TVM-FFI object: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :name: ABI.Object.Destroy + :start-after: [Object.Destroy.begin] + :end-before: [Object.Destroy.end] + +Inheritance Checking +~~~~~~~~~~~~~~~~~~~~ + +TVM-FFI models single inheritance as a tree where each node points to its parent. +Each type has a unique type index, and the system tracks ancestors, inheritance depth, and other metadata. +This information is available via :cpp:func:`TVMFFIGetTypeInfo`. + +The following C code checks whether a type is a subclass of another: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Object.IsInstance.begin] + :end-before: [Object.IsInstance.end] + +.. _abi-tensor: + +Tensor +------ + +.. seealso:: + + :doc:`tensor` for details about TVM-FFI tensors and DLPack interoperability. + +TVM-FFI provides :cpp:class:`tvm::ffi::TensorObj`, a DLPack-native tensor class that is also a standard TVM-FFI object. +This means tensors can be managed using the same reference counting mechanisms as other objects. + +.. dropdown:: C ABI Reference: :cpp:class:`tvm::ffi::TensorObj` + :icon: code + + .. code-block:: cpp + :caption: tvm/ffi/container/tensor.h + + class TensorObj : public Object, public DLTensor { + // no other members besides those from Object and DLTensor + }; + + +Access Tensor Metadata +~~~~~~~~~~~~~~~~~~~~~~ + +The following C code obtains a :c:struct:`DLTensor` pointer from a :cpp:class:`~tvm::ffi::TensorObj`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Tensor.AccessDLTensor.begin] + :end-before: [Tensor.AccessDLTensor.end] + +The :c:struct:`DLTensor` pointer provides access to shape, dtype, device, data pointer, and other tensor metadata. + +Construct Tensor from DLPack +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following C code constructs a :cpp:class:`~tvm::ffi::TensorObj` from a :c:struct:`DLManagedTensorVersioned`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Tensor_FromDLPack.begin] + :end-before: [Tensor_FromDLPack.end] + +.. hint:: + TVM-FFI's Python API automatically wraps framework tensors (e.g., :py:class:`torch.Tensor`) as :cpp:class:`~tvm::ffi::TensorObj`, + so manual conversion is typically unnecessary. + +Destruct Tensor +~~~~~~~~~~~~~~~ + +As a standard TVM-FFI object, :cpp:class:`~tvm::ffi::TensorObj` follows the :ref:`standard destruction pattern `. +When the reference count reaches zero, the deleter callback (:cpp:member:`TVMFFIObject::deleter`) executes. + +Export Tensor to DLPack +~~~~~~~~~~~~~~~~~~~~~~~ + +To share a :cpp:class:`~tvm::ffi::TensorObj` with other frameworks, export it as a :c:struct:`DLManagedTensorVersioned`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Tensor_ToDLPackVersioned.begin] + :end-before: [Tensor_ToDLPackVersioned.end] + +Note that the caller takes ownership of the returned :c:struct:`DLManagedTensorVersioned* ` +and must call its ``deleter`` to release the tensor. + +.. _abi-function: + +Function +-------- + +.. seealso:: + + :ref:`sec:function` for a detailed description of TVM-FFI functions. + +All functions in TVM-FFI follow a unified C calling convention that enables ABI-stable, +type-erased, and cross-language function calls, defined by :cpp:type:`TVMFFISafeCallType`. + +**Calling convention.** The signature includes: + +- ``handle`` (``void*``): Optional resource handle passed to the callee; typically ``NULL`` for exported symbols +- ``args`` (``TVMFFIAny*``) and ``num_args`` (``int``): Array of non-owning :cpp:class:`~tvm::ffi::AnyView` input arguments +- ``result`` (``TVMFFIAny*``): Owning :cpp:class:`~tvm::ffi::Any` output value +- Return value: ``0`` for success; ``-1`` or ``-2`` for errors (see :ref:`sec:exception`) + +See :ref:`sec:function-calling-convention` for more details. + +**Memory layout.** The :cpp:class:`~tvm::ffi::FunctionObj` stores call pointers after the object header. + +.. dropdown:: C ABI Reference: :cpp:class:`TVMFFIFunctionCell` + :icon: code + + .. literalinclude:: ../../include/tvm/ffi/c_api.h + :language: c + :start-after: [TVMFFIFunctionCell.begin] + :end-before: [TVMFFIFunctionCell.end] + :caption: tvm/ffi/c_api.h + +Construct and Destroy +~~~~~~~~~~~~~~~~~~~~~ + +.. important:: + Dynamic function creation is useful for passing lambdas or closures across language boundaries. + +The following C code constructs a :cpp:class:`~tvm::ffi::FunctionObj` from a :cpp:type:`TVMFFISafeCallType` and a ``deleter`` callback. +The ``deleter`` cleans up resources owned by the function; for global symbols, it is typically ``NULL``. + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Function.Construct.begin] + :end-before: [Function.Construct.end] + +Release a :cpp:class:`~tvm::ffi::FunctionObj` using the :ref:`standard destruction pattern `. + +Global Registry +~~~~~~~~~~~~~~~ + +**Retrieve a global function.** The following C code uses :cpp:func:`TVMFFIFunctionGetGlobal` to retrieve a function by name from the global registry: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Function.GetGlobal.begin] + :end-before: [Function.GetGlobal.end] + +.. note:: + :cpp:func:`TVMFFIFunctionGetGlobal` returns an owning handle. + The caller must release it by calling :cpp:func:`TVMFFIObjectDecRef` when it's no longer needed. + +**Register a global function.** The following C code uses :cpp:func:`TVMFFIFunctionSetGlobal` to register a function by name in the global registry: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Function.SetGlobal.begin] + :end-before: [Function.SetGlobal.end] + +Call Function +~~~~~~~~~~~~~ + +The following C code invokes a :cpp:class:`~tvm::ffi::FunctionObj` with arguments: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Function.Call.begin] + :end-before: [Function.Call.end] + + +.. _abi-exception: + +Exception +--------- + +.. seealso:: + + :ref:`sec:exception` for detailed exception handling patterns. + +Exceptions are a central part of TVM-FFI's ABI and calling convention. +When errors occur, they are stored as objects with a :cpp:class:`TVMFFIErrorCell` payload. + +.. dropdown:: C ABI Reference: :cpp:class:`TVMFFIErrorCell` + :icon: code + + .. literalinclude:: ../../include/tvm/ffi/c_api.h + :language: c + :start-after: [TVMFFIErrorCell.begin] + :end-before: [TVMFFIErrorCell.end] + :caption: tvm/ffi/c_api.h + +.. important:: + Errors from all languages (e.g. Python, C++) will be properly translated into the TVM-FFI error object. + + +Retrieve Error Object +~~~~~~~~~~~~~~~~~~~~~ + +When a function returns ``-1``, an error object is stored in thread-local storage (TLS). +Retrieve it with :cpp:func:`TVMFFIErrorMoveFromRaised`, which returns a :cpp:class:`tvm::ffi::ErrorObj`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Error.HandleReturnCode.begin] + :end-before: [Error.HandleReturnCode.end] + +This function transfers ownership to the caller and clears the TLS slot. +Call :cpp:func:`TVMFFIObjectDecRef` when done to avoid memory leaks. + +**Frontend errors (-2).** Error code ``-2`` is reserved for frontend errors. +It is returned when :cpp:func:`TVMFFIEnvCheckSignals` detects a pending Python signal. +In this case, do not retrieve the error from TLS; instead, consult the frontend's error mechanism. + +.. admonition:: Print Error Message + :class: hint + + The error payload is a :cpp:type:`TVMFFIErrorCell` structure containing the error kind, message, and backtrace. + Access it by skipping the :cpp:type:`TVMFFIObject` header via pointer arithmetic. + + .. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Error.Print.begin] + :end-before: [Error.Print.end] + + This prints the error message along with its backtrace. + +Raise Exception +~~~~~~~~~~~~~~~ + +The following C code sets the TLS error and returns ``-1`` via :cpp:func:`TVMFFIErrorSetRaisedFromCStr`: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Error.RaiseException.begin] + :end-before: [Error.RaiseException.end] + +For non-null-terminated strings, use :cpp:func:`TVMFFIErrorSetRaisedFromCStrParts`, which accepts explicit string lengths. + +.. note:: + You rarely need to create a :cpp:class:`~tvm::ffi::ErrorObj` directly. + The C APIs :cpp:func:`TVMFFIErrorSetRaisedFromCStr` and :cpp:func:`TVMFFIErrorSetRaisedFromCStrParts` handle this internally. + +.. _abi-string-and-byte: + +🚧 String and Bytes +------------------- + +.. warning:: + This section is under construction. + +The ABI supports strings and bytes as first-class citizens. A string can take multiple forms that are identified by +its ``type_index``. + +- ``kTVMFFIRawStr``: raw C string terminated by ``\0``. +- ``kTVMFFISmallStr``: small string, the length is stored in ``small_str_len`` and data is stored in ``v_bytes``. +- ``kTVMFFIStr``: on-heap string object for strings that are longer than 7 characters. + +The following code shows the layout of the on-heap string object. + +.. code-block:: cpp + + // span-like data structure to store header and length + typedef struct { + const char* data; + size_t size; + } TVMFFIByteArray; + + // showcase the layout of the on-heap string. + class StringObj : public ffi::Object, public TVMFFIByteArray { + }; + + +The following code shows how to read a string from :cpp:class:`TVMFFIAny` + +.. code-block:: cpp + + TVMFFIByteArray ReadString(const TVMFFIAny *value) { + TVMFFIByteArray ret; + if (value->type_index == kTVMFFIRawStr) { + ret.data = value->v_c_str; + ret.size = strlen(ret.data); + } else if (value->type_index == kTVMFFISmallStr) { + ret.data = value->v_bytes; + ret.size = value->small_str_len; + } else { + assert(value->type_index == kTVMFFIStr); + ret = *reinterpret_cast( + reinterpret_cast(value->v_obj) + sizeof(TVMFFIObject)); + } + return ret; + } + + +Similarly, we have type indices to represent bytes. The C++ API provides classes +:cpp:class:`~tvm::ffi::String` and :cpp:class:`~tvm::ffi::Bytes` to enable the automatic conversion of these values with Any storage format. + +**Rationales**. Separate string and bytes enable clear mappings from the Python side. Small string allows us to +store short names on-stack. To favor 8-byte alignment (v_bytes) and keep things simple, we did not further +pack characters into the ``small_len`` field. + +Further Reading +--------------- + +- :doc:`any`: High-level C++ usage of :cpp:class:`~tvm::ffi::Any` and :cpp:class:`~tvm::ffi::AnyView` +- :doc:`object_and_class`: The object system and reflection +- :doc:`tensor`: Tensor classes and DLPack interoperability +- :doc:`func_module`: Functions, exceptions, and modules +- :doc:`../get_started/stable_c_abi`: Quick introduction to the stable C ABI diff --git a/docs/concepts/any.rst b/docs/concepts/any.rst index efc6652a..7f234ce1 100644 --- a/docs/concepts/any.rst +++ b/docs/concepts/any.rst @@ -27,9 +27,7 @@ values of a wide variety of types, including primitives, objects, and strings. Unlike ``std::any``, it is designed for zero-copy inter-language exchange without RTTI, featuring a fixed 16-byte layout with built-in reference counting and ownership semantics. -This tutorial covers everything you need to know about :cpp:class:`~tvm::ffi::Any` and :cpp:class:`~tvm::ffi::AnyView`: -common usage patterns, ownership semantics, and memory layout. - +This tutorial covers common usage patterns, ownership semantics, and memory layout. Common Usage ------------ @@ -169,6 +167,8 @@ Compare with ``nullptr`` to check for ``None``: } +.. _any-ownership: + Ownership --------- @@ -195,8 +195,8 @@ The core distinction between :cpp:class:`tvm::ffi::Any` and - Function inputs - Return values, storage -Code Examples -~~~~~~~~~~~~~~ +Examples +~~~~~~~~ :cpp:class:`~tvm::ffi::AnyView` is a lightweight, non-owning view. Copying it simply copies 16 bytes with no reference count updates, making it ideal for passing arguments without overhead: @@ -245,25 +245,9 @@ Destruction Semantics in C In C, which lacks RAII, you must manually destroy :cpp:class:`~tvm::ffi::Any` objects by calling :cpp:func:`TVMFFIObjectDecRef` for heap-allocated objects. +Destroying an :cpp:class:`~tvm::ffi::AnyView` is effectively a no-op - just clear its contents. -.. code-block:: cpp - - void destroy_any(TVMFFIAny* any) { - if (any->type_index >= kTVMFFIStaticObjectBegin) { - // Decrement the reference count of the heap-allocated object - TVMFFIObjectDecRef(any->v_obj); - } - *any = (TVMFFIAny){0}; - } - -In contrast, destroying an :cpp:class:`~tvm::ffi::AnyView` is effectively a no-op - just clear its contents. - -.. code-block:: cpp - - void destroy_any_view(TVMFFIAny* any_view) { - *any_view = (TVMFFIAny){0}; - } - +See :ref:`abi-destruct-any` for C code examples. Layout ------ @@ -310,6 +294,8 @@ It is effectively a layout-stable 16-byte tagged union. * The first 4 bytes (:cpp:member:`TVMFFIAny::type_index`) serve as a tag identifying the stored type. * The last 8 bytes hold the actual value - either stored inline for atomic types (e.g., ``int64_t``, ``float64``, ``void*``) or as a pointer to a heap-allocated object. +.. _any-atomic-types: + Atomic Types ~~~~~~~~~~~~ @@ -371,6 +357,8 @@ Note that raw pointers like :c:struct:`DLTensor* ` and ``char*`` also These pointers carry no ownership, so the caller must ensure the pointed-to data outlives the :cpp:class:`~tvm::ffi::AnyView` or :cpp:class:`~tvm::ffi::Any`. +.. _any-heap-allocated-objects: + Heap-Allocated Objects ~~~~~~~~~~~~~~~~~~~~~~ @@ -431,7 +419,7 @@ inline using **small string optimization**, avoiding heap allocation entirely: Further Reading --------------- -- **Object system**: :doc:`object_and_class` covers how TVM-FFI objects work, including reference counting and type checking -- **Function system**: :doc:`func_module` covers function calling conventions and the global registry -- **C examples**: :doc:`../get_started/stable_c_abi` demonstrates working with :cpp:class:`TVMFFIAny` directly in C -- **Tensor conversions**: :doc:`tensor` covers how tensors flow through :cpp:class:`~tvm::ffi::Any` and :cpp:class:`~tvm::ffi::AnyView` +- :doc:`object_and_class`: How TVM-FFI objects work, including reference counting and type checking +- :doc:`func_module`: Function calling conventions and the global registry +- :doc:`tensor`: How tensors flow through :cpp:class:`~tvm::ffi::Any` and :cpp:class:`~tvm::ffi::AnyView` +- :doc:`abi_overview`: Low-level C ABI details for working with :cpp:class:`TVMFFIAny` directly diff --git a/docs/concepts/func_module.rst b/docs/concepts/func_module.rst index 1024b917..b4a6142d 100644 --- a/docs/concepts/func_module.rst +++ b/docs/concepts/func_module.rst @@ -15,20 +15,20 @@ specific language governing permissions and limitations under the License. -Function, Exception and Module -============================== +Function and Module +=================== TVM-FFI provides a unified and ABI-stable calling convention that enables cross-language function calls between C++, Python, Rust, and other languages. Functions are first-class :doc:`TVM-FFI objects `. -This tutorial covers everything you need to know about defining, registering, -and calling TVM-FFI functions, their exception handling, and working with modules. +This tutorial covers defining, registering, and calling TVM-FFI functions, +exception handling, and working with modules. Glossary -------- -TVM-FFI ABI. :cpp:type:`TVMFFISafeCallType` +TVM-FFI ABI, or "Packed Function". :cpp:type:`TVMFFISafeCallType` A stable C calling convention where every function is represented by a single signature, which enables type-erased, cross-language function calls. This calling convention is used across all TVM-FFI function calls at the ABI boundary. @@ -190,17 +190,15 @@ to a :py:class:`tvm_ffi.Function` at the ABI boundary. The example below demonst print(func_add(1, 2)) -Exception ABI -------------- +.. _sec:function: -This section describes the exception handling contract in the TVM-FFI Stable C ABI. -Exceptions are first-class citizens in TVM-FFI, and this section specifies: +Function +-------- -- How to properly throw exceptions from a TVM-FFI ABI function -- How to check for and propagate exceptions from a TVM-FFI ABI function +.. _sec:function-calling-convention: -TVM-FFI C ABI -~~~~~~~~~~~~~ +Calling Convention +~~~~~~~~~~~~~~~~~~ All TVM-FFI functions ultimately conform to the :cpp:type:`TVMFFISafeCallType` signature, which provides a stable C ABI for cross-language calls. The C calling convention is defined as: @@ -222,145 +220,43 @@ that the caller must zero-initialize before the call. **Return value**. The ABI returns an **error code** that indicates: -- ``0``: Success -- ``-1``: Error occurred, retrievable with :cpp:func:`TVMFFIErrorMoveFromRaised` -- ``-2``: Very rare frontend error +- **Error code 0**: Success +- **Error code -1**: Error occurred, retrievable with :cpp:func:`TVMFFIErrorMoveFromRaised` +- **Error code -2**: Very rare frontend error .. hint:: See :doc:`Any ` for more details on the semantics of :cpp:type:`tvm::ffi::AnyView` and :cpp:type:`tvm::ffi::Any`. -Retrieve Errors in C -~~~~~~~~~~~~~~~~~~~~ - -When a TVM-FFI function returns a non-zero code, it indicates that an error occurred -and a :cpp:class:`tvm::ffi::ErrorObj` is stored in thread-local storage (TLS). -This section shows how to retrieve the error object and print the error message and backtrace. - -.. note:: - - An :cpp:class:`~tvm::ffi::ErrorObj` is a :cpp:class:`~tvm::ffi::Object` with a :cpp:class:`TVMFFIErrorCell` payload - as defined below: - - .. code-block:: cpp - - typedef struct { - TVMFFIByteArray kind; // Error type (e.g., "ValueError") - TVMFFIByteArray message; // Error message - TVMFFIByteArray backtrace; // Stack trace (most-recent call first) - void (*update_backtrace)(...); // Hook to append/replace backtrace - } TVMFFIErrorCell; +This design is called a **packed function**, because it "packs" all arguments into a single array of type-erased :cpp:type:`tvm::ffi::AnyView`, +and further unifies calling convention across all languages without resorting to JIT compilation. -**Print an Error**. The example code below shows how to print an error message and backtrace. +More specifically, this mechanism enables the following scenarios: -.. code-block:: cpp - - #include +- **Dynamic languages**. Well-optimized bindings are provided for, e.g. Python, to translate arguments into packed function format, and translate return value back to the host language. +- **Static languages**. Metaprogramming techniques, such as C++ templates, are usually available to directly instantiate packed format on stack, saving the need for dynamic examination. +- **Cross-language callbacks**. Language-agnostic :cpp:class:`tvm::ffi::Function` makes it easy to call between languages without depending on language-specific features such as GIL. - void PrintError(TVMFFIObject* err) { - TVMFFIErrorCell* cell = (TVMFFIErrorCell*)((char*)err + sizeof(TVMFFIObject)); - fprintf(stderr, "%.*s: %.*s\n", (int)cell->kind.size, cell->kind.data, (int)cell->message.size, cell->message.data); - if (cell->backtrace.size) { - fprintf(stderr, "Backtrace:\n%.*s\n", (int)cell->backtrace.size, cell->backtrace.data); - } - } - -The payload of the error object is a :cpp:type:`TVMFFIErrorCell` structure -containing the error kind, message, and backtrace. It can be accessed -by skipping the :cpp:type:`TVMFFIObject` header using pointer arithmetic. - -**Retrieve the error object**. When the error code is ``-1``, the error object is stored in TLS -and can be retrieved with :cpp:func:`TVMFFIErrorMoveFromRaised`. - -.. code-block:: cpp - - void HandleReturnCode(int rc) { - TVMFFIObject* err = NULL; - if (rc == 0) { - // Success - } else if (rc == -1) { - // Move the raised error from TLS (clears TLS slot) - TVMFFIErrorMoveFromRaised(&err); // now `err` owns the error object - if (err != NULL) { - PrintError(err); // print the error - TVMFFIObjectDecRef(err); // Release the error object - } - } else if (rc == -2) { - // Frontend (e.g., Python) already has an exception set. - // Do not fetch from TLS; consult the frontend's error mechanism. - } - } - -This function transfers ownership of the error object to the caller and clears the TLS slot. -You must call :cpp:func:`TVMFFIObjectDecRef` to release the object when done to avoid memory leaks. - -**Rare frontend errors**. Error code ``-2`` is reserved for rare frontend errors. It is returned only -when the C API :cpp:func:`TVMFFIEnvCheckSignals` returns non-zero during execution, indicating that -the Python side has a pending signal requiring attention. In this case, the caller should not fetch -the error object from TLS but instead consult the frontend's error mechanism to handle the exception. - -Raise Errors in C -~~~~~~~~~~~~~~~~~ - -As part of TVM-FFI's calling convention, returning ``-1`` indicates that an error occurred -and the error object is stored in the TLS slot. The error object can contain arbitrary -user-defined information, such as error messages, backtraces, or Python frame-local variables. - -.. hint:: - Compiler code generation may use similar patterns to raise errors in generated code. +**Performance Implications**. This approach is in practice highly efficient in machine learning workloads. -The example below sets the TLS error and returns ``-1`` using :cpp:func:`TVMFFIErrorSetRaisedFromCStr`: +- In Python/C++ calls, we can get to microsecond level overhead, which is generally similar to overhead for eager mode; +- When both sides of calls are static languages, the overhead will go down to tens of nanoseconds. -.. code-block:: cpp - - #include - - int __tvm_ffi_my_kernel(void* handle, const TVMFFIAny* args, - int32_t num_args, TVMFFIAny* result) { - // Validate inputs - if (num_args < 2) { - TVMFFIErrorSetRaisedFromCStr("ValueError", "Expected at least 2 arguments"); - return -1; - } - // ... kernel implementation ... - return 0; - } - -Alternatively, :cpp:func:`TVMFFIErrorSetRaisedFromCStrParts` accepts explicit string lengths, -which is useful when the error kind and message are not null-terminated. - -**Propagating errors**. For chains of generated calls, simply propagate return codes—TLS carries -the error details: - -.. code-block:: cpp - - int outer_function(...) { - int err_code = 0; - - err_code = inner_function(...); - if (err_code != 0) goto RAII; // Propagate error; TLS has the details - - RAII: - // clean up owned resources - return err_code; - } +.. note:: + Although we found it less necessary in practice, further link time optimization (LTO) is still theoretically possible + in scenarios where both sides are static languages with a known symbol and linked into a single binary. + In this case, the callee can be inlined into caller side and the stack argument memory can be passed into register passing. -Function --------- +.. _sec:function-layout: Layout and ABI ~~~~~~~~~~~~~~ :cpp:class:`tvm::ffi::FunctionObj` stores two call pointers in :cpp:class:`TVMFFIFunctionCell`: -.. code-block:: cpp - - typedef struct { - TVMFFISafeCallType safe_call; - void* cpp_call; - } TVMFFIFunctionCell; +- ``safe_call``: Used for cross-ABI function calls; intercepts exceptions and stores them in TLS. +- ``cpp_call``: Used within the same DSO; exceptions are thrown directly for better performance. -``safe_call`` is used for cross-ABI function calls: it intercepts exceptions and stores them in TLS. -``cpp_call`` is used within the same DSO, where exceptions are thrown directly for better performance. +See :ref:`abi-function` for the C struct definition. .. important:: @@ -407,40 +303,35 @@ calling convention. in :c:macro:`TVM_FFI_SAFE_CALL_BEGIN` / :c:macro:`TVM_FFI_SAFE_CALL_END` macros. -C Registry APIs -~~~~~~~~~~~~~~~ +Compiler developers commonly need to look up global functions in generated code. +Use :cpp:func:`TVMFFIFunctionGetGlobal` to retrieve a function by name, then call it with :cpp:func:`TVMFFIFunctionCall`. +See :ref:`abi-function` for C code examples. -.. list-table:: - :header-rows: 1 - :widths: 40 60 +.. _sec:exception: - * - C API - - Description - * - :cpp:func:`TVMFFIFunctionGetGlobal` - - Get a function by name; returns an owning handle. - * - :cpp:func:`TVMFFIFunctionSetGlobal` - - Register a function in the global registry. - * - :cpp:func:`TVMFFIFunctionCall` - - Call a function with the given arguments. +Exception +~~~~~~~~~ -Compiler developers commonly need to look up global functions in generated code. Use -:cpp:func:`TVMFFIFunctionGetGlobal` to retrieve a function by name, then call it with :cpp:func:`TVMFFIFunctionCall`. -The example below demonstrates how to look up and call a global function in C: +This section describes the exception handling contract in the TVM-FFI Stable C ABI. +Exceptions are first-class citizens in TVM-FFI, and this section specifies: -.. code-block:: cpp +- How to properly throw exceptions from a TVM-FFI ABI function +- How to check for and propagate exceptions from a TVM-FFI ABI function - int LookupAndCall(const char* global_function_name, const TVMFFIAny* args, int num_args, TVMFFIAny* result) { - TVMFFIObject* func = NULL; - int err_code; - if ((err_code = TVMFFIFunctionGetGlobal(global_function_name, &func)) != 0) - goto RAII; - if ((err_code = TVMFFIFunctionCall(func, args, num_args, result)) != 0) - goto RAII; - - RAII: // clean up owned resources - if (func != NULL) TVMFFIObjectDecRef(func); - return err_code; - } +When a TVM-FFI function returns a non-zero code, an error occurred. +An :cpp:class:`~tvm::ffi::ErrorObj` is stored in thread-local storage (TLS) and can be retrieved +with :cpp:func:`TVMFFIErrorMoveFromRaised`. + +- **Error code -1:** Retrieve the error from TLS, print it, and release via :cpp:func:`TVMFFIObjectDecRef`. +- **Error code -2:** A rare frontend error; consult the frontend's error mechanism instead of TLS. + +To raise an error, use :cpp:func:`TVMFFIErrorSetRaisedFromCStr` to set the TLS error and return ``-1``. +For chains of calls, simply propagate return codes - TLS carries the error details. + +See :ref:`abi-exception` for C code examples. + + +.. _sec:module: Modules ------- @@ -560,5 +451,5 @@ Further Reading - :doc:`any`: How functions are stored in :cpp:class:`~tvm::ffi::Any` containers - :doc:`object_and_class`: The object system that backs :cpp:class:`~tvm::ffi::FunctionObj` +- :doc:`abi_overview`: Low-level C ABI details for functions and exceptions - :doc:`../packaging/python_packaging`: Packaging functions for Python wheels -- :doc:`abi_overview`: Low-level ABI details for the function calling convention diff --git a/docs/concepts/object_and_class.rst b/docs/concepts/object_and_class.rst index 7b3e5b07..91ce14ba 100644 --- a/docs/concepts/object_and_class.rst +++ b/docs/concepts/object_and_class.rst @@ -27,9 +27,7 @@ and :cpp:class:`tvm::ffi::ObjectRef`, which together form the foundation for: - **Reflection-based class exposure** across programming languages - **Serialization and deserialization** via reflection metadata -This tutorial covers everything you need to know about defining, using, and extending -TVM-FFI objects across languages. - +This tutorial covers defining, using, and extending TVM-FFI objects across languages. Glossary -------- @@ -253,106 +251,48 @@ and :cpp:func:`tvm::ffi::ObjectRef::get` to convert a managed reference to a raw ABI and Layout -------------- -Stable C Layout -~~~~~~~~~~~~~~~ - -All subclasses of :cpp:class:`tvm::ffi::Object` share a common 24-byte header (:cpp:class:`TVMFFIObject`): - -.. code-block:: cpp - - typedef struct { - uint64_t combined_ref_count; // Bytes 0-7: strong + weak ref counts - int32_t type_index; // Bytes 8-11: runtime type identifier - uint32_t __padding; // Bytes 12-15: alignment padding - void (*deleter)(void*, int); // Bytes 16-23: destructor callback - } TVMFFIObject; - - -It is designed with the following components: - -- Reference counting and deleter callback, which are used to manage the lifetime of the object; -- Type index, which is used to interact with type registration system for type checking and casting. - -:cpp:class:`tvm::ffi::ObjectRef` and :cpp:class:`tvm::ffi::ObjectPtr` are smart pointers whose -layout is equivalent to: +**Stable C Layout**. All subclasses of :cpp:class:`tvm::ffi::Object` share a common 24-byte header (:cpp:class:`TVMFFIObject`) +containing reference counts, type index, and a deleter callback. +See :ref:`abi-object` for the C struct definition. :cpp:class:`tvm::ffi::ObjectRef` and :cpp:class:`tvm::ffi::ObjectPtr` are smart pointers +equivalent to a single ``void*`` pointer. -.. code-block:: cpp - - struct { void* data; }; +.. _object-reference-counting: Reference Counting ~~~~~~~~~~~~~~~~~~ -**Deleter action**. When an object is managed by :cpp:class:`~tvm::ffi::ObjectRef`, the ``deleter`` callback is invoked: +.. seealso:: -- When strong reference count reaches zero: the object's destructor is called. -- When weak reference count reaches zero: the memory is freed. - -The flags in :cpp:enum:`TVMFFIObjectDeleterFlagBitMask` indicate which action to perform. + :ref:`abi-object-ownership` for C code examples. **Intrusive reference counting**. The reference count is stored directly in the object header, not in a separate control block. -This design reduces memory overhead and improves cache locality. Specifically, the :cpp:member:`TVMFFIObject::combined_ref_count` -field stores a 64-bit integer that packs both strong and weak reference counts: +This design reduces memory overhead and improves cache locality. The :cpp:member:`TVMFFIObject::combined_ref_count` +field packs both strong (lower 32 bits) and weak (upper 32 bits) reference counts in a single 64-bit integer. -.. code-block:: cpp +C APIs are provided to manipulate the reference count: - // Strong ref count: lower 32 bits - uint32_t strong_ref_count = combined_ref_count & 0xFFFFFFFF; - // Weak ref count: upper 32 bits - uint32_t weak_ref_count = (combined_ref_count >> 32) & 0xFFFFFFFF; +- :cpp:func:`TVMFFIObjectIncRef` to increase the strong reference count +- :cpp:func:`TVMFFIObjectDecRef` to decrease the strong reference count -C APIs are provided to manipulate the reference count of an object: +**Deleter**. When an object is managed by :cpp:class:`~tvm::ffi::ObjectRef`, the ``deleter`` callback is invoked: -- :cpp:func:`TVMFFIObjectIncRef` to increase the strong reference count; -- :cpp:func:`TVMFFIObjectDecRef` to decrease the strong reference count. +- When strong reference count reaches zero: the object's destructor is called. +- When weak reference count reaches zero: the memory is freed. +The flags in :cpp:enum:`TVMFFIObjectDeleterFlagBitMask` indicate which action to perform. .. _object-conversion-with-any: -Conversion between :cpp:class:`~tvm::ffi::Any` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -At the stable C ABI boundary, TVM-FFI passes values using :cpp:class:`Any ` (owning) -or :cpp:class:`AnyView ` (non-owning). Object handles are stored in the -:cpp:member:`TVMFFIAny::v_obj` field with a type index >= ``kTVMFFIStaticObjectBegin``. - -**Any/AnyView to Object**. Extract an object handle from :cpp:class:`TVMFFIAny`: - -.. code-block:: cpp - - // Converts Any/AnyView to Object handle (non-owning) - int AnyToObjectPtr(const TVMFFIAny* value, TVMFFIObject** out) { - if (value->type_index >= kTVMFFIStaticObjectBegin) { - *out = (TVMFFIObject*)(value->v_obj); - return SUCCESS; - } - return FAILURE; // Not an object type - } - -**Object to AnyView**. Store an object handle into non-owning :cpp:class:`AnyView `: +**Conversion with Any**. At the stable C ABI boundary, TVM-FFI passes values using :cpp:class:`Any ` +(owning) or :cpp:class:`AnyView ` (non-owning). Object handles are stored in the +:cpp:member:`TVMFFIAny::v_obj` field with a type index >= :cpp:enumerator:`kTVMFFIStaticObjectBegin `. -.. code-block:: cpp - - // Converts Object handle to AnyView (non-owning) - void ObjectToAnyView(TVMFFIObject* obj, int32_t type_index, TVMFFIAny* out) { - out->type_index = type_index; - out->zero_padding = 0; - out->v_obj = obj; - } - -**Object to Any**. Store an object handle into owning :cpp:class:`Any `. -The function increments the reference count to take shared ownership. - -.. code-block:: cpp - - // Converts Object handle to Any (owning, increments refcount) - void ObjectToAny(TVMFFIObject* obj, int32_t type_index, TVMFFIAny* out) { - ObjectToAnyView(obj, type_index, out); - TVMFFIObjectIncRef(obj); // Take ownership - } +See :ref:`abi-object-ownership` for C code examples demonstrating: -Later, release ownership by calling :cpp:func:`TVMFFIObjectDecRef` on :cpp:member:`TVMFFIAny::v_obj`. +- Extracting an object handle from :cpp:class:`TVMFFIAny` +- Storing an object handle into :cpp:class:`~tvm::ffi::Any` or :cpp:class:`~tvm::ffi::AnyView` +- Managing ownership via reference counting Object Type Registry -------------------- @@ -477,5 +417,5 @@ Further Reading - :doc:`any`: How objects are stored in :cpp:class:`~tvm::ffi::Any` containers - :doc:`func_module`: Function objects and the global registry - :doc:`tensor`: Tensor objects and DLPack interoperability -- :doc:`../packaging/python_packaging`: Packaging C++ objects for Python -- :doc:`abi_overview`: Low-level ABI details for the object system +- :doc:`abi_overview`: Low-level C ABI details for the object system +- :doc:`../packaging/python_packaging`: Packaging C++ objects for Python wheels diff --git a/docs/concepts/tensor.rst b/docs/concepts/tensor.rst index 22123f08..d7f93430 100644 --- a/docs/concepts/tensor.rst +++ b/docs/concepts/tensor.rst @@ -34,11 +34,7 @@ and minimal extensions for ownership management. they provide safer and more convenient abstractions over raw DLPack structs. -This tutorial is organized as follows: - -* **Common Usage**: the most important tensor APIs, including allocation and stream handling. -* **Tensor Classes**: what tensor types are provided and which one you should use. -* **Conversion between TVMFFIAny**: how tensors flow across ABI boundaries. +This tutorial covers common usage patterns, tensor classes, and how tensors flow across ABI boundaries. Glossary -------- @@ -351,142 +347,30 @@ it does not include: Conversion between :cpp:class:`TVMFFIAny` ----------------------------------------- -At the stable C ABI boundary, TVM-FFI passes values using an "Any-like" carrier - either -:cpp:class:`Any ` (owning) or :cpp:class:`AnyView ` (non-owning). -These are 128-bit tagged unions derived from :cpp:class:`TVMFFIAny` that contain: - -* a :cpp:member:`type_index ` that indicates the type of the payload, and -* a union payload that may contain: - - * A1. Primitive values, such as integers, floats, enums, raw pointers, or - * A2. TVM-FFI object handles, which are reference-counted pointers. +At the stable C ABI boundary, TVM-FFI passes values using :cpp:class:`Any ` (owning) +or :cpp:class:`AnyView ` (non-owning). Tensors have two possible representations: -Specifically for tensors stored in :cpp:class:`Any ` or :cpp:class:`AnyView `, -there are two possible representations: +* **Non-owning:** :c:struct:`DLTensor* ` with type index :cpp:enumerator:`TVMFFITypeIndex::kTVMFFIDLTensorPtr` +* **Owning:** :cpp:class:`TensorObj* ` with type index :cpp:enumerator:`TVMFFITypeIndex::kTVMFFITensor` -* Non-owning views as A1 (primitive values), i.e. :c:struct:`DLTensor* ` whose type index is :cpp:enumerator:`TVMFFITypeIndex::kTVMFFIDLTensorPtr`. -* Owning objects as A2 (TVM-FFI tensor object handles), i.e., :cpp:class:`TensorObj* ` whose type index is :cpp:enumerator:`TVMFFITypeIndex::kTVMFFITensor`. - -Therefore, when you see a tensor in :cpp:class:`Any ` or :cpp:class:`AnyView `, -first check its :cpp:member:`type_index ` to determine whether it is a raw pointer or an object handle -before converting it to the desired tensor type. +When extracting a tensor from :cpp:class:`TVMFFIAny`, check the :cpp:member:`type_index ` +to determine the representation before conversion. .. important:: - As a rule of thumb, an owning object can be converted to a non-owning view, but not vice versa. - -To Non-Owning Tensor -~~~~~~~~~~~~~~~~~~~~ - -This converts an owning :cpp:class:`Any ` or non-owning :cpp:class:`AnyView ` into a non-owning tensor. -Two type indices can be converted to a non-owning tensor view: - -- :cpp:enumerator:`TVMFFITypeIndex::kTVMFFIDLTensorPtr`: the payload is a raw pointer :c:struct:`DLTensor* `. -- :cpp:enumerator:`TVMFFITypeIndex::kTVMFFITensor`: the payload is a TVM-FFI tensor object handle, from which you can extract the underlying :c:struct:`DLTensor` according to the layout defined in :ref:`Figure 1 `. - -The snippets below are plain C (C99-compatible) and assume the TVM-FFI C ABI definitions from -``tvm/ffi/c_api.h`` are available. - -.. code-block:: cpp - - // Converts Any/AnyView to DLTensor* - int AnyToDLTensorView(const TVMFFIAny* value, DLTensor** out) { - if (value->type_index == kTVMFFIDLTensorPtr) { - *out = (DLTensor*)value->v_ptr; - return SUCCESS; - } - if (value->type_index == kTVMFFITensor) { - // See Figure 1 for layout of tvm::ffi::TensorObj - TVMFFIObject* obj = value->v_obj; - *out = (DLTensor*)((char*)obj + sizeof(TVMFFIObject)); - return SUCCESS; - } - return FAILURE; - } - -:cpp:class:`TensorView ` can be constructed directly from the returned :c:struct:`DLTensor* `. - -To Owning Tensor -~~~~~~~~~~~~~~~~ - -This converts an owning :cpp:class:`Any ` or non-owning :cpp:class:`AnyView ` into an owning :cpp:class:`TensorObj `. Only type index :cpp:enumerator:`TVMFFITypeIndex::kTVMFFITensor` can be converted to an owning tensor because it contains a TVM-FFI tensor object handle. The conversion involves incrementing the reference count to take ownership. - -.. code-block:: cpp - - // Converts Any/AnyView to TensorObj* - int AnyToOwnedTensor(const TVMFFIAny* value, TVMFFIObjectHandle* out) { - if (value->type_index == kTVMFFITensor) { - *out = (TVMFFIObjectHandle)value->v_obj; - return SUCCESS; - } - return FAILURE; - } + An owning tensor can be converted to a non-owning view, but not vice versa. -The caller can obtain shared ownership by calling :cpp:func:`TVMFFIObjectIncRef` on the returned handle, -and later release it with :cpp:func:`TVMFFIObjectDecRef`. +See :ref:`abi-tensor` for C code examples demonstrating: -From Owning Tensor -~~~~~~~~~~~~~~~~~~ - -This converts an owning :cpp:class:`TensorObj ` to an owning :cpp:class:`Any ` or non-owning :cpp:class:`AnyView `. It sets the type index to :cpp:enumerator:`TVMFFITypeIndex::kTVMFFITensor` and stores the tensor object handle in the payload. - -.. code-block:: cpp - - // Converts TensorObj* to AnyView - int TensorToAnyView(TVMFFIObjectHandle tensor, TVMFFIAny* out_any_view) { - out_any_view->type_index = kTVMFFITensor; - out_any_view->zero_padding = 0; - out_any_view->v_obj = (TVMFFIObject*)tensor; - return SUCCESS; - } - - // Converts TensorObj* to Any - int TensorToAny(TVMFFIObjectHandle tensor, TVMFFIAny* out_any) { - TVMFFIAny any_view; - int ret = TensorToAnyView(tensor, &any_view); - if (ret != SUCCESS) { - return ret; - } - TVMFFIObjectIncRef(tensor); - *out_any = any_view; - return SUCCESS; - } - -The C API :cpp:func:`TVMFFIObjectIncRef` obtains shared ownership of the tensor into `out_any`. Later, release it with -:cpp:func:`TVMFFIObjectDecRef` on its :cpp:member:`TVMFFIAny::v_obj` field. - -From Non-Owning Tensor -~~~~~~~~~~~~~~~~~~~~~~ - -This converts a non-owning :cpp:class:`TensorView ` to non-owning :cpp:class:`AnyView `. -It sets the type index to :cpp:enumerator:`TVMFFITypeIndex::kTVMFFIDLTensorPtr` and stores a raw pointer to :c:struct:`DLTensor* ` in the payload. - -.. warning:: - - Non-owning :c:struct:`DLTensor` or :cpp:class:`TensorView ` can be converted to non-owning :cpp:class:`AnyView `, but cannot be converted to owning :cpp:class:`Any `. - -.. code-block:: cpp - - // Converts DLTensor* to AnyView - int DLTensorToAnyView(DLTensor* tensor, TVMFFIAny* out) { - out->type_index = kTVMFFIDLTensorPtr; - out->zero_padding = 0; - out->v_ptr = tensor; - return SUCCESS; - } - - // Converts TensorView to AnyView - int TensorViewToAnyView(const tvm::ffi::TensorView& tensor_view, TVMFFIAny* out) { - return DLTensorToAnyView(tensor_view.GetDLTensorPtr(), out); - } +- Extracting a :c:struct:`DLTensor` pointer from :cpp:class:`TVMFFIAny` +- Constructing a :cpp:class:`~tvm::ffi::TensorObj` from DLPack +- Exporting a :cpp:class:`~tvm::ffi::TensorObj` to DLPack Further Reading --------------- -- :cpp:class:`TensorObj ` and :cpp:class:`Tensor ` are part of the standard TVM-FFI object system. - See :doc:`object_and_class` for a comprehensive guide, or :ref:`Object Storage Format ` for low-level layout details. -- :cpp:class:`AnyView ` and :cpp:class:`Any ` are part of the stable C ABI. - Tutorial :doc:`Stable C ABI<../get_started/stable_c_abi>` explains the ABI design at a high level, - and :doc:`ABI Overview ` shares details on the design. -- DLPack specification can be found at :external+data-api:doc:`DLPack protocol `, and documentation at :external+dlpack:doc:`C API ` and :external+dlpack:doc:`Python API `. -- Kernel library developers may also refer to :doc:`../guides/kernel_library_guide` and `FlashInfer `_ for best practices on building high-performance kernel libraries with TVM-FFI. +- :doc:`object_and_class`: The object system that backs :cpp:class:`~tvm::ffi::TensorObj` +- :doc:`any`: How tensors are stored in :cpp:class:`~tvm::ffi::Any` containers +- :doc:`abi_overview`: Low-level C ABI details for tensor conversion +- :doc:`../guides/kernel_library_guide`: Best practices for building kernel libraries with TVM-FFI +- :external+dlpack:doc:`DLPack C API `: The underlying tensor interchange standard diff --git a/docs/guides/compiler_integration.md b/docs/guides/compiler_integration.md index 254df38d..a9b816a6 100644 --- a/docs/guides/compiler_integration.md +++ b/docs/guides/compiler_integration.md @@ -93,7 +93,7 @@ Some of the key takeaways include: - Use return value for error handling, set error via {cpp:func}`TVMFFIErrorSetRaisedFromCStr` or {cpp:func}`TVMFFIErrorSetRaisedFromCStrParts`. -You can also check out the [ABI overview](../concepts/abi_overview.md) for a more complete guide. +You can also check out the [ABI overview](../concepts/abi_overview.rst) for a more complete guide. ## Graph Compilers diff --git a/docs/index.rst b/docs/index.rst index ef2b2b8d..31c5efe3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,7 +58,7 @@ Table of Contents :maxdepth: 1 :caption: Concepts - concepts/abi_overview.md + concepts/abi_overview.rst concepts/any.rst concepts/object_and_class.rst concepts/tensor.rst diff --git a/examples/abi_overview/example_code.c b/examples/abi_overview/example_code.c new file mode 100644 index 00000000..b8f3c19a --- /dev/null +++ b/examples/abi_overview/example_code.c @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// NOLINTBEGIN(modernize-deprecated-headers,modernize-use-nullptr,clang-diagnostic-c99-extensions,clang-diagnostic-missing-field-initializers,clang-diagnostic-unused-but-set-variable,clang-analyzer-deadcode.DeadStores,clang-diagnostic-unused-variable,clang-diagnostic-deprecated) +#include +#include +#include +#include +#include + +int IS_OWNING_ANY = 1; + +// [Any_AnyView.FromInt_Float.begin] +TVMFFIAny Any_AnyView_FromInt(int64_t value) { + TVMFFIAny any; + any.type_index = kTVMFFIInt; + any.zero_padding = 0; + any.v_int64 = value; + return any; +} + +TVMFFIAny Any_AnyView_FromFloat(double value) { + TVMFFIAny any; + any.type_index = kTVMFFIFloat; + any.zero_padding = 0; + any.v_float64 = value; + return any; +} +// [Any_AnyView.FromInt_Float.end] + +// [Any_AnyView.FromObjectPtr.begin] +TVMFFIAny Any_AnyView_FromObjectPtr(TVMFFIObject* obj) { + TVMFFIAny any; + assert(obj != NULL); + any.type_index = kTVMFFIObject; + any.zero_padding = 0; + any.v_obj = obj; + // Increment refcount if it's Any (owning) instead of AnyView (borrowing) + if (IS_OWNING_ANY) { + TVMFFIObjectIncRef(obj); + } + return any; +} +// [Any_AnyView.FromObjectPtr.end] + +// [Any_AnyView.Destroy.begin] +void Any_AnyView_Destroy(TVMFFIAny* any) { + if (IS_OWNING_ANY) { + // Checks if `any` holds a heap-allocated object, + // and if so, decrements the reference count + if (any->type_index >= kTVMFFIStaticObjectBegin) { + TVMFFIObjectDecRef(any->v_obj); + } + } + *any = (TVMFFIAny){0}; // Clears the `any` struct +} +// [Any_AnyView.Destroy.end] + +// [Any_AnyView.GetInt_Float.begin] +int64_t Any_AnyView_GetInt(const TVMFFIAny* any) { + if (any->type_index == kTVMFFIInt || any->type_index == kTVMFFIBool) { + return any->v_int64; + } else if (any->type_index == kTVMFFIFloat) { + return (int64_t)(any->v_float64); + } + assert(0); // FAILED to read int + return 0; +} + +double Any_AnyView_GetFloat(const TVMFFIAny* any) { + if (any->type_index == kTVMFFIInt || any->type_index == kTVMFFIBool) { + return (double)(any->v_int64); + } else if (any->type_index == kTVMFFIFloat) { + return any->v_float64; + } + assert(0); // FAILED to read float + return 0.0; +} +// [Any_AnyView.GetInt_Float.end] + +// [Any_AnyView.GetDLTensor.begin] +DLTensor* Any_AnyView_GetDLTensor(const TVMFFIAny* value) { + if (value->type_index == kTVMFFIDLTensorPtr) { + return (DLTensor*)(value->v_ptr); + } else if (value->type_index == kTVMFFITensor) { + return (DLTensor*)((char*)(value->v_obj) + sizeof(TVMFFIObject)); + } + assert(0); // FAILED to read DLTensor + return NULL; +} +// [Any_AnyView.GetDLTensor.end] + +// [Any_AnyView.GetObject.begin] +TVMFFIObject* Any_AnyView_GetObject(const TVMFFIAny* value) { + if (value->type_index == kTVMFFINone) { + return NULL; // Handling nullptr if needed + } else if (value->type_index >= kTVMFFIStaticObjectBegin) { + return value->v_obj; + } + assert(0); // FAILED: not a TVM-FFI object + return NULL; +} +// [Any_AnyView.GetObject.end] + +// [Object.IsInstance.begin] +int Object_IsInstance(int32_t sub_type_index, int32_t super_type_index, int32_t super_type_depth) { + const TVMFFITypeInfo* sub_type_info = NULL; + // Everything is a subclass of object. + if (sub_type_index == super_type_index) { + return 1; + } + // Invariance: parent index is always smaller than the child. + if (sub_type_index < super_type_index) { + return 0; + } + sub_type_info = TVMFFIGetTypeInfo(sub_type_index); + return sub_type_info->type_depth > super_type_depth && + sub_type_info->type_ancestors[super_type_depth]->type_index == super_type_index; +} +// [Object.IsInstance.end] + +// [Object.MoveFromAny.begin] +void Object_MoveFromAny(TVMFFIAny* any, TVMFFIObject** obj) { + assert(any->type_index >= kTVMFFIStaticObjectBegin); + *obj = any->v_obj; + (*any) = (TVMFFIAny){0}; + if (!IS_OWNING_ANY) { + TVMFFIObjectIncRef(*obj); + } +} +// [Object.MoveFromAny.end] + +// [Object.Destroy.begin] +void Object_Destroy(TVMFFIObject* obj) { + assert(obj != NULL); + TVMFFIObjectDecRef(obj); +} +// [Object.Destroy.end] + +// [Tensor.AccessDLTensor.begin] +DLTensor* Tensor_AccessDLTensor(TVMFFIObject* tensor) { + assert(tensor != NULL); + return (DLTensor*)((char*)tensor + sizeof(TVMFFIObject)); +} +// [Tensor.AccessDLTensor.end] + +// [Tensor_FromDLPack.begin] +TVMFFIObject* Tensor_FromDLPack(DLManagedTensorVersioned* from) { + int err_code = 0; + TVMFFIObject* out = NULL; + err_code = TVMFFITensorFromDLPackVersioned(from, // input DLPack tensor + /*require_alignment=*/0, // no alignment requirement + /*require_contiguous=*/1, // require contiguous tensor + (void**)(&out)); + assert(err_code == 0); + return out; +} +// [Tensor_FromDLPack.end] + +// [Tensor_ToDLPackVersioned.begin] +DLManagedTensorVersioned* Tensor_ToDLPackVersioned(TVMFFIObject* tensor) { + int err_code = 0; + DLManagedTensorVersioned* out = NULL; + err_code = TVMFFITensorToDLPackVersioned(tensor, &out); + assert(err_code == 0); + return out; +} +// [Tensor_ToDLPackVersioned.end] + +// [Function.Construct.begin] +TVMFFIObject* Function_Construct(void* self, TVMFFISafeCallType safe_call, + void (*deleter)(void* self)) { + int err_code; + TVMFFIObject* out = NULL; + err_code = TVMFFIFunctionCreate(self, safe_call, deleter, (void**)(&out)); + assert(err_code == 0); + return out; +} +// [Function.Construct.end] + +// [Function.Call.begin] +int64_t CallFunction(TVMFFIObject* func, int64_t x, int64_t y) { + int err_code; + TVMFFIAny args[2]; + TVMFFIAny result = (TVMFFIAny){0}; + args[0] = Any_AnyView_FromInt(x); + args[1] = Any_AnyView_FromInt(y); + err_code = TVMFFIFunctionCall(func, args, 2, &result); + assert(err_code == 0); + return Any_AnyView_GetInt(&result); +} +// [Function.Call.end] + +// [Function.GetGlobal.begin] +TVMFFIObject* Function_RetrieveGlobal(const char* name) { + TVMFFIObject* out = NULL; + TVMFFIByteArray name_byte_array = {name, strlen(name)}; + int err_code = TVMFFIFunctionGetGlobal(&name_byte_array, (void**)(&out)); + assert(err_code == 0); + return out; +} +// [Function.GetGlobal.end] + +// [Function.SetGlobal.begin] +void Function_SetGlobal(const char* name, TVMFFIObject* func) { + TVMFFIByteArray name_byte_array = {name, strlen(name)}; + int err_code = TVMFFIFunctionSetGlobal(&name_byte_array, func, 0); + assert(err_code == 0); +} +// [Function.SetGlobal.end] + +// [Error.Print.begin] +void PrintError(TVMFFIObject* err) { + TVMFFIErrorCell* cell = (TVMFFIErrorCell*)((char*)err + sizeof(TVMFFIObject)); + fprintf(stderr, "%.*s: %.*s\n", // + (int)cell->kind.size, cell->kind.data, // e.g. "ValueError" + (int)cell->message.size, cell->message.data); // e.g. "Expected at least 2 arguments" + if (cell->backtrace.size) { + fprintf(stderr, "Backtrace:\n%.*s\n", (int)cell->backtrace.size, cell->backtrace.data); + } +} +// [Error.Print.end] + +// [Error.HandleReturnCode.begin] +void Error_HandleReturnCode(int rc) { + TVMFFIObject* err = NULL; + if (rc == -1) { + // Move the raised error from TLS (clears TLS slot) + TVMFFIErrorMoveFromRaised((void**)(&err)); // now `err` owns the error object + if (err != NULL) { + PrintError(err); // print the error + // IMPORTANT: Release the error object, or gets memory leaks + TVMFFIObjectDecRef(err); + } + } else if (rc == -2) { + // Frontend (e.g., Python) already has an exception set. + // Do not fetch from TLS; consult the frontend's error mechanism. + return; + } +} +// [Error.HandleReturnCode.end] + +// [Error.RaiseException.begin] +int Error_RaiseException(void* handle, const TVMFFIAny* args, int32_t num_args, TVMFFIAny* result) { + TVMFFIErrorSetRaisedFromCStr("ValueError", "Expected at least 2 arguments"); + return -1; +} +// [Error.RaiseException.end] + +// NOLINTEND(modernize-deprecated-headers,modernize-use-nullptr,clang-diagnostic-c99-extensions,clang-diagnostic-missing-field-initializers,clang-diagnostic-unused-but-set-variable,clang-analyzer-deadcode.DeadStores,clang-diagnostic-unused-variable,clang-diagnostic-deprecated) +int main() { return 0; } diff --git a/include/tvm/ffi/c_api.h b/include/tvm/ffi/c_api.h index 4da0d4d0..ae3a3ce0 100644 --- a/include/tvm/ffi/c_api.h +++ b/include/tvm/ffi/c_api.h @@ -81,6 +81,7 @@ typedef struct { uint32_t patch; } TVMFFIVersion; +// [TVMFFITypeIndex.begin] #ifdef __cplusplus enum TVMFFITypeIndex : int32_t { #else @@ -184,6 +185,7 @@ typedef enum { #else } TVMFFITypeIndex; #endif +// [TVMFFITypeIndex.end] /*! \brief Handle to Object from C API's pov */ typedef void* TVMFFIObjectHandle; @@ -218,6 +220,7 @@ typedef enum { } TVMFFIObjectDeleterFlagBitMask; #endif +// [TVMFFIObject.begin] /*! * \brief C-based type of all FFI object header that allocates on heap. */ @@ -268,7 +271,9 @@ typedef struct { }; #endif } TVMFFIObject; +// [TVMFFIObject.end] +// [TVMFFIAny.begin] /*! * \brief C-based type of all on stack Any value. * @@ -321,7 +326,9 @@ typedef struct { }; #endif } TVMFFIAny; +// [TVMFFIAny.end] +// [TVMFFIByteArray.begin] /*! * \brief Byte array data structure used by String and Bytes. * @@ -349,6 +356,7 @@ typedef struct { /*! \brief The size of the data. */ size_t size; } TVMFFIShapeCell; +// [TVMFFIByteArray.end] /*! * \brief Mode to update the backtrace of the error. @@ -366,6 +374,7 @@ typedef enum { } TVMFFIBacktraceUpdateMode; #endif +// [TVMFFIErrorCell.begin] /*! * \brief Error cell used in error object following header. */ @@ -405,7 +414,9 @@ typedef struct { */ TVMFFIObjectHandle extra_context; } TVMFFIErrorCell; +// [TVMFFIErrorCell.end] +// [TVMFFISafeCallType.begin] /*! * \brief Type that defines C-style safe call convention * @@ -441,7 +452,9 @@ typedef struct { */ typedef int (*TVMFFISafeCallType)(void* handle, const TVMFFIAny* args, int32_t num_args, TVMFFIAny* result); +// [TVMFFISafeCallType.end] +// [TVMFFIFunctionCell.begin] /*! * \brief Object cell for function object following header. */ @@ -462,6 +475,7 @@ typedef struct { */ void* cpp_call; } TVMFFIFunctionCell; +// [TVMFFIFunctionCell.end] /*! * \brief Object cell for opaque object following header. From e38261dae36467f164c6fa7103db70fcf3556fb6 Mon Sep 17 00:00:00 2001 From: Junru Shao Date: Tue, 13 Jan 2026 10:56:32 -0800 Subject: [PATCH 2/2] save --- docs/concepts/abi_overview.rst | 30 ++++++++++++++++++--- docs/concepts/func_module.rst | 3 +++ examples/abi_overview/example_code.c | 39 +++++++++++++++++++++++----- include/tvm/ffi/c_api.h | 2 +- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/docs/concepts/abi_overview.rst b/docs/concepts/abi_overview.rst index 56291ebf..89848cc1 100644 --- a/docs/concepts/abi_overview.rst +++ b/docs/concepts/abi_overview.rst @@ -274,10 +274,11 @@ The following C code obtains a :c:struct:`DLTensor` pointer from a :cpp:class:`~ The :c:struct:`DLTensor` pointer provides access to shape, dtype, device, data pointer, and other tensor metadata. -Construct Tensor from DLPack -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Construct Tensor +~~~~~~~~~~~~~~~~ -The following C code constructs a :cpp:class:`~tvm::ffi::TensorObj` from a :c:struct:`DLManagedTensorVersioned`: +**Zero-copy conversion.** The following C code constructs a :cpp:class:`~tvm::ffi::TensorObj` from a :c:struct:`DLManagedTensorVersioned`, +which shares the underlying data buffer without allocating new memory. .. literalinclude:: ../../examples/abi_overview/example_code.c :language: c @@ -288,6 +289,26 @@ The following C code constructs a :cpp:class:`~tvm::ffi::TensorObj` from a :c:st TVM-FFI's Python API automatically wraps framework tensors (e.g., :py:class:`torch.Tensor`) as :cpp:class:`~tvm::ffi::TensorObj`, so manual conversion is typically unnecessary. +**Allocate new memory.** Alternatively, if memory allocation is intended, the following C code constructs a :cpp:class:`~tvm::ffi::TensorObj` from a :c:struct:`DLTensor` pointer: + +.. literalinclude:: ../../examples/abi_overview/example_code.c + :language: c + :start-after: [Tensor_Alloc.begin] + :end-before: [Tensor_Alloc.end] + +The ``prototype`` contains the shape, dtype, device, and other tensor metadata that will be used to allocate the new tensor. +And the allocator, by default, is the framework's (e.g., PyTorch) allocator, which is automatically set when importing the framework. + +To override or explicitly look up the allocator, use :cpp:func:`TVMFFIEnvSetDLPackManagedTensorAllocator` and :cpp:func:`TVMFFIEnvGetDLPackManagedTensorAllocator`. + +.. warning:: + In kernel library usecases, it is usually not recommended to dynamically allocate tensors inside a kernel, and instead always pre-allocate outputs, + and pass them as :cpp:class:`~tvm::ffi::TensorView` parameters. This approach + + - avoids memory fragmentation and performance pitfalls, + - prevents CUDA graph incompatibilities on GPU, and + - allows the outer framework to control allocation policy (pools, device strategies, etc.). + Destruct Tensor ~~~~~~~~~~~~~~~ @@ -328,6 +349,9 @@ type-erased, and cross-language function calls, defined by :cpp:type:`TVMFFISafe See :ref:`sec:function-calling-convention` for more details. +.. important:: + The caller must zero-initialize the output argument ``result`` before the call. + **Memory layout.** The :cpp:class:`~tvm::ffi::FunctionObj` stores call pointers after the object header. .. dropdown:: C ABI Reference: :cpp:class:`TVMFFIFunctionCell` diff --git a/docs/concepts/func_module.rst b/docs/concepts/func_module.rst index b4a6142d..d3540231 100644 --- a/docs/concepts/func_module.rst +++ b/docs/concepts/func_module.rst @@ -218,6 +218,9 @@ specified by ``args`` and ``num_args``. **Output argument**. The output argument ``result`` is an owning :cpp:type:`tvm::ffi::Any` that the caller must zero-initialize before the call. +.. important:: + The caller must zero-initialize the output argument ``result`` before the call. + **Return value**. The ABI returns an **error code** that indicates: - **Error code 0**: Success diff --git a/examples/abi_overview/example_code.c b/examples/abi_overview/example_code.c index b8f3c19a..049ea935 100644 --- a/examples/abi_overview/example_code.c +++ b/examples/abi_overview/example_code.c @@ -16,13 +16,26 @@ * specific language governing permissions and limitations * under the License. */ - -// NOLINTBEGIN(modernize-deprecated-headers,modernize-use-nullptr,clang-diagnostic-c99-extensions,clang-diagnostic-missing-field-initializers,clang-diagnostic-unused-but-set-variable,clang-analyzer-deadcode.DeadStores,clang-diagnostic-unused-variable,clang-diagnostic-deprecated) +/* + * Example code for TVM-FFI ABI overview. + * + * Compilation command: + * + * ```bash + * gcc $(tvm-ffi-config --cflags) \ + * $(tvm-ffi-config --ldflags) \ + * $(tvm-ffi-config --libs) \ + * -Wl,-rpath,$(tvm-ffi-config --libdir) \ + * -o ./example_code + * ``` + */ +// NOLINTBEGIN(modernize-deprecated-headers,modernize-use-nullptr) #include #include #include #include #include +#include int IS_OWNING_ANY = 1; @@ -164,15 +177,27 @@ DLTensor* Tensor_AccessDLTensor(TVMFFIObject* tensor) { TVMFFIObject* Tensor_FromDLPack(DLManagedTensorVersioned* from) { int err_code = 0; TVMFFIObject* out = NULL; - err_code = TVMFFITensorFromDLPackVersioned(from, // input DLPack tensor - /*require_alignment=*/0, // no alignment requirement - /*require_contiguous=*/1, // require contiguous tensor - (void**)(&out)); + err_code = TVMFFITensorFromDLPackVersioned( // + from, // input DLPack tensor + /*require_alignment=*/0, // no alignment requirement + /*require_contiguous=*/1, // require contiguous tensor + (void**)(&out)); assert(err_code == 0); return out; } // [Tensor_FromDLPack.end] +// [Tensor_Alloc.begin] +TVMFFIObject* Tensor_Alloc(DLTensor* prototype) { + int err_code = 0; + TVMFFIObject* out = NULL; + assert(prototype->data == NULL); + err_code = TVMFFIEnvTensorAlloc(prototype, (void**)(&out)); + assert(err_code == 0); + return out; +} +// [Tensor_Alloc.end] + // [Tensor_ToDLPackVersioned.begin] DLManagedTensorVersioned* Tensor_ToDLPackVersioned(TVMFFIObject* tensor) { int err_code = 0; @@ -263,5 +288,5 @@ int Error_RaiseException(void* handle, const TVMFFIAny* args, int32_t num_args, } // [Error.RaiseException.end] -// NOLINTEND(modernize-deprecated-headers,modernize-use-nullptr,clang-diagnostic-c99-extensions,clang-diagnostic-missing-field-initializers,clang-diagnostic-unused-but-set-variable,clang-analyzer-deadcode.DeadStores,clang-diagnostic-unused-variable,clang-diagnostic-deprecated) +// NOLINTEND(modernize-deprecated-headers,modernize-use-nullptr) int main() { return 0; } diff --git a/include/tvm/ffi/c_api.h b/include/tvm/ffi/c_api.h index ae3a3ce0..c02075bf 100644 --- a/include/tvm/ffi/c_api.h +++ b/include/tvm/ffi/c_api.h @@ -62,7 +62,7 @@ /*! \brief TVM FFI minor version. */ #define TVM_FFI_VERSION_MINOR 1 /*! \brief TVM FFI patch version. */ -#define TVM_FFI_VERSION_PATCH 8 +#define TVM_FFI_VERSION_PATCH 9 // NOLINTEND(modernize-macro-to-enum) #ifdef __cplusplus