From 25f5e64b2ca0652c5f28d354eade7e367226ad09 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 11 Nov 2025 11:20:56 +0200 Subject: [PATCH] api: packet: improve dynamic and static packet references Current API regarding dynamic packet references is not easy to efficiently support in some ODP implementations with HW accelerated packet output due to the need to update reference counts after TX completion and based on that conditionally free packet buffers. odp_packet_ref() has been implemented only as a full packet copy in all ODP implementations. Redefine the semantics of odp_packet_ref() and the packets involved in packet data sharing in a way that hopefully makes it simpler to support packet data sharing with restrictions in packet output and with per-packet reference counting instead of per-segment reference counting. The new semantics also enable use cases that were not previously possible. Specify better the rules regarding packet data layout manipulation operations, allowing private packet data also in the tail of the packets that reference other packets. Pull and truncate operations may involve shared packet data, in which case they just remove that data from the packet, not affecting the packet owning the data. All extend and truncate operations always produce non-shared packet data. Change odp_packet_has_ref() to indicate whether a packet is referenced by another packet, not whether the packet shares data with another packet. In particular, odp_packet_ref() returns a packet that shares data with the given packet but is not itself referenced by other packets. Specify that packet data and layout of packets referenced by other packets cannot be modified at all. Add new odp_packet_is_referencing() to test whether a packet references another packet. Introduce pktio capability bits to indicate whether a pktio supports sending packets with shared data without using the dont_free flag. Sending with the dont_free flag is supported with all types of packets. Signed-off-by: Janne Peltonen --- include/odp/api/spec/ipsec.h | 6 +- include/odp/api/spec/packet.h | 217 ++++++++++++++++++------- include/odp/api/spec/packet_io.h | 12 ++ include/odp/api/spec/packet_io_types.h | 48 +++++- include/odp/api/spec/traffic_mngr.h | 36 +++- 5 files changed, 257 insertions(+), 62 deletions(-) diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index 1bde8798e1..19b323a75a 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2016-2018 Linaro Limited - * Copyright (c) 2021-2022 Nokia + * Copyright (c) 2021-2025 Nokia */ /** @@ -434,6 +434,10 @@ int odp_ipsec_out_enq(const odp_packet_t pkt[], int num, * transformed packets to the specified output interface (or tm_queue), instead of * generating events for those. * + * Packets that reference or are referenced by other packets may be sent + * using this function only if the pktout to be used has the relevant + * capability. See odp_pktio_capability_t::packet_ref. XXX dontfree + * * Inline operation parameters are defined per packet. The array of parameters * must have 'num' elements and is pointed to by 'inline_param'. * diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index e817ee133f..911569b1e4 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2013-2018 Linaro Limited - * Copyright (c) 2021-2024 Nokia + * Copyright (c) 2021-2025 Nokia */ /** @@ -94,6 +94,13 @@ int odp_packet_alloc_multi(odp_pool_t pool, uint32_t len, * * Frees the packet into the packet pool it was allocated from. * + * A packet being referenced by other packets (see odp_packet_ref() and + * odp_packet_ref_static()) continues to consume pool capacity until there + * are no more references to it left. Referencing packets are not affected + * when a packet they reference is freed. + * + * The freed packet may not be used in any way after this call. + * * @param pkt Packet handle */ void odp_packet_free(odp_packet_t pkt); @@ -909,6 +916,25 @@ uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg); * the destination packet. The source packet may have been allocated from * any pool. * + * 'dst' must not be a referenced packet or a static packet reference. It can + * be a referencing packet. 'src' may be any kind of packet. The returned + * packet will be either a normal packet or referencing packet depending + * on the input packets and implementation details as follows: + * + * If both the destination packet and the source packet are normal packets, + * the result packet will also be a normal packet. If the destination packet + * is a referencing packet, the result packet will also be a referencing packet. + * Otherwise the result packet is either a normal or referencing packet + * depending on the packets and the implementation. + * + * dst src result + * ------------------------------------------------- + * normal normal normal + * normal referenced normal or referencing + * normal referencing normal or referencing + * normal static ref normal or referencing + * referencing any referencing + * * On failure, both handles remain valid and packets are not modified. * * @param[in, out] dst Pointer to destination packet handle. A successful @@ -1109,6 +1135,9 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool, odp_packet_buf_t pkt_buf[], * e.g., packet retransmissions. Use odp_packet_ref() or odp_packet_ref_pkt() * for more flexible, dynamic references. * + * This function may be called simultaneously in different threads for the + * same packet. + * * Packet is not modified on failure. * * @param pkt Handle of the packet for which a static reference is @@ -1120,65 +1149,122 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool, odp_packet_buf_t pkt_buf[], odp_packet_t odp_packet_ref_static(odp_packet_t pkt); /** - * Create a reference to a packet - * - * Returns a new (dynamic) reference to a packet starting the shared part of - * the data at a specified byte offset. Metadata and data before the offset - * are not shared with other references of the packet. The rest of the data is - * shared and must be treated as read only. Initially the returned reference - * has metadata initialized to default values and does not contain unshared - * data. Packet (head) manipulation functions may be used normally to, e.g., - * add a unique header onto the shared payload. The shared part of the packet - * may be modified again when there is a single reference left. Static and - * dynamic references must not be mixed. Results are undefined if these - * restrictions are not observed. - * - * The packet handle 'pkt' may itself be a (dynamic) reference to a packet. - * - * If the caller does not intend to modify either the packet or the new - * reference to it, odp_packet_ref_static() may be used to create - * a static reference that is more optimized for that use case. - * - * Packet is not modified on failure. - * - * @param pkt Handle of the packet for which a reference is to be - * created. - * - * @param offset Byte offset in the packet at which the shared part is to - * begin. This must be in the range 0 ... odp_packet_len(pkt)-1. - * - * @return New reference to the packet + * Create a packet that references packet data of another packet + * + * Return a packet that shares packet data with the given packet. 'pkt' + * becomes a 'referenced packet' and the returned packet will be a + * 'referencing packet'. The referencing packet is allocated from the + * same pool as the referenced packet. + * + * The packet passed to this function must not be a referencing packet or + * a static reference (see odp_packet_ref_static()). It can already be + * a referenced packet. + * + * Packet metadata is not shared. The new packet has its metadata initialized + * to the default values. + * + * Shared packet data starts at the given offset of the referenced packet + * and continues to the end of the referenced packet. The referencing packet + * may also contain private, non-shared packet data after it has been modified + * using packet manipulation operation such as push and extend (see below). + * The referencing packet may also be modified to not actually contain any + * packet data of the referenced packet. + * + * Packet data and data layout of the referenced packet must not be modified + * as long as any packet that references the packet exists. In particular, + * a referenced packet must not be passed to odp_packet_{push,pull,extend, + * trunc}_{head,tail}(), odp_packet_split(), or as the destination packet to + * odp_packet_concat() or to any other ODP API function that could modify + * packet data or packet data layout. + * + * odp_packet_has_ref() returns 1 for the referenced packet as long as it is + * being referenced by other packets and 0 for the newly created referencing + * packet. odp_packet_is_referencing() returns 1 for the created referencing + * packet. + * + * The referenced and referencing packets can be freed in any order. + * ODP ensures that the packets or packet segments backing the shared packet + * data is put back to the packet pool only after there are no references left. + * See odp_packet_free(). + * + * Packet data and packet data layout of the returned packet can be modified + * as long as the packet data bytes that are shared are not written to. Such + * packet manipulation operations do not affect the referenced packet or its + * layout. + * + * Packet data layout manipulation functions can be used for the returned + * packet (but not for the referenced packet) as follows: + * + * - odp_packet_{pull,trunc}_{head,tail}() functions can be used normally. + * Pulling and truncation can include shared packet data, in which case + * the shared packet data area shrinks. It is possible to pull or truncate + * the packet so much that it no longer shares any packet data, but that + * does not stop the referencing relationship between the packets and + * the restrictions that apply to the referencing and referenced packet + * still hold. + * + * - odp_packet_{push,extend}_{head,tail}() functions can be used normally. + * The new packet data created by the functions is always private, even if + * extending after truncating or pulling shared data. Because of this, + * odp_packet_headroom() and odp_packet_tailroom() likely return 0 for the + * newly created packet even if the referenced packet has headroom or + * tailroom. + * + * - odp_packet_concat() can be used normally but concatenating packet data + * to a destination packet that shares data with another packet may require + * packet data copying if also the source packet shares data with another + * packet. This is because, depending on the implementation, a packet may + * be able to reference only one other packet. + * + * Packets returned by this function can be used normally in ODP API functions + * unless otherwise noted. There are restrictions in packet output. + * + * Referenced packets can be used in ODP API functions that treat packet data + * and layout as read-only. Referenced packets can be used in ODP API functions + * that consume packets, unless otherwise noted. Consuming a packet behaves + * the same way as freeing a packet with respect to shared packet data. + * + * This function may be called simultaneously in different threads for the + * same packet. + * + * @param pkt Handle of the packet to which a reference is to be created. + * + * @param offset Byte offset in 'pkt' at which the shared part is to + * begin. This must be in the range 0 ... odp_packet_len(pkt)-1. + * + * @return Handle of the newly created referencing packet * @retval ODP_PACKET_INVALID On failure */ odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset); /** - * Create a reference to a packet with a header packet + * Create a packet with a header and shared data + * + * This operation is identical to ref = odp_packet_ref(pkt, offset) followed + * by odp_packet_concat(&hdr, ref). The same rules and restrictions apply to + * this combined operation as to those operations. The resulting packet + * consists of metadata and packet data of 'hdr' as unshared data, followed + * by the packet data shared with 'pkt'. * - * This operation is otherwise identical to odp_packet_ref(), but it prepends - * a supplied 'hdr' packet as the head of the new reference. The resulting - * packet consists metadata and data of the 'hdr' packet, followed by the - * shared part of packet 'pkt'. + * 'pkt' must not be a static reference or a referencing packet but it + * may be a referenced packet. 'hdr' must not be a static reference or a + * referenced packet. 'pkt' and 'hdr' must not be the same packet. * - * The packet handle ('pkt') may itself be a (dynamic) reference to a packet, - * but the header packet handle ('hdr') must be unique. Both packets must be - * have been allocated from the same pool and the handles must not refer to - * the same packet. Results are undefined if these restrictions are not - * observed. + * This function may be called simultaneously in different threads with the + * same 'pkt' but not with the same 'hdr'. * * Packets are not modified on failure. The header packet 'hdr' is consumed - * on success. + * and 'pkt' becomes a referenced packet on success. * - * @param pkt Handle of the packet for which a reference is to be + * @param pkt Handle of the packet to which a reference is to be * created. * * @param offset Byte offset in 'pkt' at which the shared part is to * begin. Must be in the range 0 ... odp_packet_len(pkt)-1. * - * @param hdr Handle of the header packet to be prefixed onto the new - * reference. Must be a unique reference. + * @param hdr Handle of the header packet to be prefixed * - * @return New reference the reference packet + * @return Handle of the newly created referencing packet * @retval ODP_PACKET_INVALID On failure */ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, @@ -1188,16 +1274,20 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, * Test if packet has multiple references * * A packet that has multiple references share data with other packets. In case - * of a static reference it also shares metadata. Shared parts must be treated - * as read only. + * of a static reference it also shares metadata. Packet data and data layout + * of packets that have references must be treated as read only. + * + * Packets created through odp_packet_ref() or odp_packet_ref_pkt() share data + * with other packets but they are not referenced by other packets. This + * function returns 0 for such packets and 1 for packet referenced by those + * packets. * - * New references are created with odp_packet_ref_static(), odp_packet_ref() and - * odp_packet_ref_pkt() calls. The intent of multiple references is to avoid - * packet copies, however some implementations may do a packet copy for some of - * the calls. If a copy is done, the new reference is actually a new, unique - * packet and this function returns '0' for it. When a real reference is - * created (instead of a copy), this function returns '1' for both packets - * (the original packet and the new reference). + * Static packet references are created using odp_packet_ref_static(). Static + * packet references reference the original packet and vice versa, so this + * function returns 1 for the original packet and all static references to + * it as long as more than one such static packet reference exists. + * + * See also odp_packet_is_referencing(). * * @param pkt Packet handle * @@ -1206,6 +1296,19 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, */ int odp_packet_has_ref(odp_packet_t pkt); +/** + * Test if a packet references another packet + * + * Return 1 if the packet references another packet, 0 otherwise. + * See odp_packet_has_ref(), odp_packet_ref(), odp_packet_ref_static() + * + * @param pkt Packet handle + * + * @retval 0 The packet is a normal packet or a referenced packet + * @retval 1 The packet is a referencing packet or a static packet reference + */ +int odp_packet_is_referencing(odp_packet_t pkt); + /* * * Copy @@ -1217,8 +1320,9 @@ int odp_packet_has_ref(odp_packet_t pkt); * Full copy of a packet * * Create a new copy of the packet. The new packet is exact copy of the source - * packet (incl. data and metadata). The pool must have been created with - * ODP_POOL_PACKET type. + * packet (incl. data and metadata). The new packet is always a normal packet, + * even if the source packet is a referencing packet. The pool must have been + * created with ODP_POOL_PACKET type. * * @param pkt Packet handle * @param pool Packet pool for allocation of the new packet. @@ -2138,8 +2242,7 @@ int odp_packet_has_tx_compl_request(odp_packet_t pkt); * (odp_pktio_capability_t.free_ctrl) for the option support. When an interface does not support * the option, it ignores the value. * - * The option must not be enabled on packets that have multiple references. The default value is - * #ODP_PACKET_FREE_CTRL_DISABLED. + * The default value is #ODP_PACKET_FREE_CTRL_DISABLED. * * @param pkt Packet handle * @param ctrl Packet free control option value diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index 371a44d851..243b55a477 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -264,6 +264,10 @@ int odp_pktin_queue(odp_pktio_t pktio, odp_pktin_queue_t queues[], int num); * odp_queue_enq_multi(). Behaviour is undefined if other events than packets * are enqueued. Application cannot dequeue from these queues. * + * Static references, referencing and referenced packets may be sent without + * the dont_free option only if the relevant packet_ref capability is set + * (see odp_pktout_packet_ref_capability_t and odp_packet_free_ctrl_set()). + * * @param pktio Packet IO handle * @param[out] queues Points to an array of queue handles for output * @param num Maximum number of queue handles to output @@ -468,6 +472,10 @@ uint64_t odp_pktin_wait_time(uint64_t nsec); * specified e.g. for protocol offload purposes. Link protocol specific frame * checksum and padding are added to frames before transmission. * + * Static references, referencing and referenced packets may be sent without + * the dont_free option only if the relevant packet_ref capability is set + * (see odp_pktout_packet_ref_capability_t and odp_packet_free_ctrl_set()). + * * @param queue Packet output queue handle for sending packets * @param packets[] Array of packets to send * @param num Number of packets to send @@ -543,6 +551,10 @@ int odp_lso_profile_destroy(odp_lso_profile_t lso_profile); * * Check LSO support level from packet IO capabilities (odp_pktio_capability_t). * + * Static references, referencing and referenced packets may be sent without + * the dont_free option only if the relevant packet_ref capability is set + * (see odp_pktout_packet_ref_capability_t and odp_packet_free_ctrl_set()). + * * @param queue Packet output queue handle * @param packet[] Array of packets to be LSO processed and sent * @param num Number of packets diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h index 1852413e4e..aea63b6f5f 100644 --- a/include/odp/api/spec/packet_io_types.h +++ b/include/odp/api/spec/packet_io_types.h @@ -519,9 +519,16 @@ typedef union odp_pktout_config_opt_t { /** Packet references not used on packet output * * When set, application indicates that it will not transmit - * packet references on this packet IO interface. - * Since every ODP implementation supports it, it is always - * ok to set this flag. + * static packet references, referenced or referencing packets + * on this packet IO interface. + * + * Even if this flag is set to zero, packets which can share + * data may not be sent unless the relevant packet I/O + * capability is set or the packet I/O supports the dont_free + * option and the option is set for the packet being sent. + * See odp_pktio_capability_t::packet_ref + * See odp_pktio_capability_t::free_ctrl + * See odp_packet_free_ctrl_set() * * 0: Packet references may be transmitted on the * interface (the default value). @@ -1010,6 +1017,38 @@ typedef struct odp_pktin_vector_capability_t { } odp_pktin_vector_capability_t; +/** Packet output capabilities regarding packets with shared data + * + * These capabilities indicate whether packets that reference other packets + * or are referenced by other packets can be sent so that they get consumed by + * the packet output. When such a packet is consumed by the packet output at + * the completion of packet transmit, the packet is handled the same as in + * odp_packet_free() (i.e. internal reference counts are updated and the packet + * segments are put back to the packet pool when there are no more references + * left). + * + * If these capabilities are not set, then the respective packet types may be + * sent only if the pktout supports the 'dont_free' option and the option is + * set for the packets being sent. In that case the packets being sent are + * not consumed nor modified. + * + * See odp_pktio_capability_t::free_ctrl + * See odp_packet_free_ctrl_set() + * See odp_packet_ref() and odp_packet_ref_static() + */ +typedef struct { + + /** Static packet references can be consumed */ + uint8_t static_ref :1; + + /** Referenced packets can be consumed */ + uint8_t referenced :1; + + /** Referencing packets can be consumed */ + uint8_t referencing :1; + +} odp_pktout_packet_ref_capability_t; + /** * Packet IO capabilities * @@ -1063,6 +1102,9 @@ typedef struct odp_pktio_capability_t { /** LSO capabilities */ odp_lso_capability_t lso; + /** Capabilities to send and consume packets that share packet data */ + odp_pktout_packet_ref_capability_t packet_ref; + /** Supported frame lengths for odp_pktio_maxlen_set() * * A frame length value of zero indicates an unsupported operation. */ diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index b3b738b9ec..bf6ca56158 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2021-2022 Nokia + * Copyright (c) 2021-2025 Nokia * Copyright (c) 2022-2025 Marvell */ @@ -433,6 +433,32 @@ typedef enum odp_tm_pkt_prio_mode { ODP_TM_PKT_PRIO_MODE_MAX, } odp_tm_pkt_prio_mode_t; +/** TM capabilities regarding packets with shared data + * + * These capabilities indicate whether packets that reference other packets + * or are referenced by other packets can be enqueued to TM. When such a + * packet is consumed by TM, either by the underlying pktout or by TM dropping + * the packet, the packet is handled the same way as in odp_packet_free() + * (i.e. internal reference counts are updated and the packet segments are put + * back to the packet pool when there are no more references left). + * + * If these capabilities are not set, then the respective packet types may not + * be enqueued to TM. + * + * See odp_packet_ref() and odp_packet_ref_static() + */ +typedef struct { + /** Static packet references can be enqueued */ + uint8_t static_ref :1; + + /** Referenced packets can be enqueued */ + uint8_t referenced :1; + + /** Referencing packets can be enqueued */ + uint8_t referencing :1; + +} odp_tm_packet_ref_capability_t; + /** TM Capabilities Record. * * The odp_tm_capabilities_t record type is used to describe the feature set @@ -609,6 +635,10 @@ typedef struct { * The value can vary between 0 and ODP_TM_MAX_PRIORITIES. */ uint8_t max_schedulers_per_node; + + /** Capabilities to process packets that share data with other packets */ + odp_tm_packet_ref_capability_t packet_ref; + } odp_tm_capabilities_t; /** Per Level Requirements @@ -2024,6 +2054,10 @@ int odp_tm_queue_disconnect(odp_tm_queue_t tm_queue); * The pkt_color bits are a result of some earlier Metering/Marking/Policing * processing. * + * Sending packets that reference or are referenced by other packets may + * only be sent if the relevant TM capabilities are set. + * See odp_tm_capabilities_t::packet_ref. + * * @param tm_queue Specifies the tm_queue (and indirectly the TM system). * @param pkt Handle to a packet. * @return Returns 0 upon success, < 0 upon failure. One of the