diff --git a/Cargo.lock b/Cargo.lock index 9c9dd90cd8..a566c28748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3023,6 +3023,7 @@ dependencies = [ "libdd-shared-runtime", "libdd-tinybytes", "libdd-trace-utils", + "rmp", "rmp-serde", "tokio-util", "tracing", diff --git a/examples/ffi/trace_exporter.c b/examples/ffi/trace_exporter.c index 3fe7a90902..5859f4556e 100644 --- a/examples/ffi/trace_exporter.c +++ b/examples/ffi/trace_exporter.c @@ -81,8 +81,81 @@ int test_error_on_panic(void) { return 0; } +int verify_structured_value_encoder(void) { + uint8_t text[] = "stable"; + uint8_t binary[] = {0x00, 0xff}; + ddog_TracerValueToken tokens[] = { + {.kind = DDOG_TRACER_VALUE_ARRAY, .child_count = 2}, + { + .kind = DDOG_TRACER_VALUE_STRING, + .bytes = {.ptr = text, .len = sizeof(text) - 1}, + }, + { + .kind = DDOG_TRACER_VALUE_BINARY, + .bytes = {.ptr = binary, .len = sizeof(binary)}, + }, + }; + ddog_TracerEncodedValue *blob = NULL; + ddog_TraceExporterError *err = ddog_tracer_encode_value( + (ddog_Slice_TracerValueToken){ + .ptr = tokens, + .len = sizeof(tokens) / sizeof(tokens[0]), + }, + &blob); + if (err) { + handle_error(err); + return 1; + } + + memset(text, 'x', sizeof(text) - 1); + memset(binary, 'x', sizeof(binary)); + static const uint8_t expected[] = { + 0x92, 0xa6, 's','t','a','b','l','e', 0xc4, 0x02, 0x00, 0xff, + }; + ddog_ByteSlice encoded = ddog_tracer_encoded_value_as_slice(blob); + int matches = encoded.len == sizeof(expected) && + memcmp(encoded.ptr, expected, sizeof(expected)) == 0; + ddog_tracer_encoded_value_free(blob); + if (!matches) { + fprintf(stderr, "Structured value encoder did not return an owned blob\n"); + return 1; + } + return 0; +} + +int verify_structured_value_encoder_rejects_invalid(void) { + ddog_TracerValueToken tokens[] = { + {.kind = 255}, + }; + ddog_TracerEncodedValue *blob = NULL; + ddog_TraceExporterError *err = ddog_tracer_encode_value( + (ddog_Slice_TracerValueToken){ + .ptr = tokens, + .len = sizeof(tokens) / sizeof(tokens[0]), + }, + &blob); + if (err == NULL) { + fprintf(stderr, "Structured value encoder accepted an unknown token kind\n"); + ddog_tracer_encoded_value_free(blob); + return 1; + } + ddog_trace_exporter_error_free(err); + + // The encoder writes the out-handle only on success, so a rejected input + // leaves the caller's NULL blob untouched and there is nothing to free. + if (blob != NULL) { + fprintf(stderr, "Structured value encoder wrote a blob on the error path\n"); + ddog_tracer_encoded_value_free(blob); + return 1; + } + return 0; +} + int main(int argc, char** argv) { + if (verify_structured_value_encoder() != 0) return 1; + if (verify_structured_value_encoder_rejects_invalid() != 0) return 1; + // Initialize logger with optional path from command line const char* log_path = (argc > 1) ? argv[1] : NULL; if (log_init(log_path) != 0) { diff --git a/libdd-data-pipeline-ffi/Cargo.toml b/libdd-data-pipeline-ffi/Cargo.toml index 01370a859c..6bc3f09e35 100644 --- a/libdd-data-pipeline-ffi/Cargo.toml +++ b/libdd-data-pipeline-ffi/Cargo.toml @@ -38,5 +38,6 @@ libdd-shared-runtime = { version = "2.0.0", path = "../libdd-shared-runtime" } libdd-common-ffi = { path = "../libdd-common-ffi", default-features = false } libdd-tinybytes = { path = "../libdd-tinybytes" } libdd-trace-utils = { path = "../libdd-trace-utils" } +rmp = { version = "0.8.14", default-features = false } tokio-util = "0.7.11" tracing.workspace = true \ No newline at end of file diff --git a/libdd-data-pipeline-ffi/cbindgen.toml b/libdd-data-pipeline-ffi/cbindgen.toml index 1214a9d4e9..6ae816c631 100644 --- a/libdd-data-pipeline-ffi/cbindgen.toml +++ b/libdd-data-pipeline-ffi/cbindgen.toml @@ -15,6 +15,7 @@ typedef struct ddog_TracerSpan ddog_TracerSpan; typedef struct ddog_TracerSpanEvent ddog_TracerSpanEvent; typedef struct ddog_TracerTraceChunks ddog_TracerTraceChunks; typedef struct ddog_TraceExporterCancelToken ddog_TraceExporterCancelToken; +typedef uint8_t ddog_TracerValueKind; """ [export] @@ -38,6 +39,15 @@ exclude = ["TraceExporter", "TracerSpan", "TracerSpanEvent", "TracerTraceChunks" "TracerSpanLinkAttribute" = "ddog_TracerSpanLinkAttribute" "TracerTraceChunks" = "ddog_TracerTraceChunks" "TokioCancellationToken" = "ddog_TraceExporterCancelToken" +"DDOG_TRACER_VALUE_NIL" = "DDOG_TRACER_VALUE_NIL" +"DDOG_TRACER_VALUE_BOOL" = "DDOG_TRACER_VALUE_BOOL" +"DDOG_TRACER_VALUE_I64" = "DDOG_TRACER_VALUE_I64" +"DDOG_TRACER_VALUE_U64" = "DDOG_TRACER_VALUE_U64" +"DDOG_TRACER_VALUE_F64" = "DDOG_TRACER_VALUE_F64" +"DDOG_TRACER_VALUE_STRING" = "DDOG_TRACER_VALUE_STRING" +"DDOG_TRACER_VALUE_BINARY" = "DDOG_TRACER_VALUE_BINARY" +"DDOG_TRACER_VALUE_ARRAY" = "DDOG_TRACER_VALUE_ARRAY" +"DDOG_TRACER_VALUE_MAP" = "DDOG_TRACER_VALUE_MAP" [export.mangle] rename_types = "PascalCase" diff --git a/libdd-data-pipeline-ffi/src/lib.rs b/libdd-data-pipeline-ffi/src/lib.rs index c8f594f391..8f6505b1fa 100644 --- a/libdd-data-pipeline-ffi/src/lib.rs +++ b/libdd-data-pipeline-ffi/src/lib.rs @@ -8,6 +8,7 @@ mod error; mod response; +mod structured_value; mod trace_exporter; mod tracer; diff --git a/libdd-data-pipeline-ffi/src/structured_value.rs b/libdd-data-pipeline-ffi/src/structured_value.rs new file mode 100644 index 0000000000..af3b571f9b --- /dev/null +++ b/libdd-data-pipeline-ffi/src/structured_value.rs @@ -0,0 +1,439 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::catch_panic; +use crate::error::{ExporterError, ExporterErrorCode as ErrorCode}; +#[cfg(all(feature = "catch_panic", panic = "unwind"))] +use crate::gen_error; +use libdd_common_ffi::slice::{AsBytes, ByteSlice, Slice}; +use rmp::encode::{ + write_array_len, write_bin, write_bool, write_f64, write_map_len, write_nil, write_sint, + write_str, write_uint, +}; +use std::ptr::NonNull; + +pub const DDOG_TRACER_VALUE_NIL: u8 = 0; +pub const DDOG_TRACER_VALUE_BOOL: u8 = 1; +pub const DDOG_TRACER_VALUE_I64: u8 = 2; +pub const DDOG_TRACER_VALUE_U64: u8 = 3; +pub const DDOG_TRACER_VALUE_F64: u8 = 4; +pub const DDOG_TRACER_VALUE_STRING: u8 = 5; +pub const DDOG_TRACER_VALUE_BINARY: u8 = 6; +pub const DDOG_TRACER_VALUE_ARRAY: u8 = 7; +pub const DDOG_TRACER_VALUE_MAP: u8 = 8; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +enum TracerValueKind { + Nil = DDOG_TRACER_VALUE_NIL, + Bool = DDOG_TRACER_VALUE_BOOL, + I64 = DDOG_TRACER_VALUE_I64, + U64 = DDOG_TRACER_VALUE_U64, + F64 = DDOG_TRACER_VALUE_F64, + String = DDOG_TRACER_VALUE_STRING, + Binary = DDOG_TRACER_VALUE_BINARY, + Array = DDOG_TRACER_VALUE_ARRAY, + Map = DDOG_TRACER_VALUE_MAP, +} + +const MAX_DEPTH: u32 = 64; + +/// One value in a flat preorder representation of a structured value. +/// +/// Scalar tokens use the corresponding scalar field. String and binary tokens +/// use `bytes`. Array and map tokens use `child_count`; a map is followed by +/// two values per entry (key, then value). All other fields must be ignored. +/// `kind` remains a raw integer so malformed C input can be rejected without +/// constructing an invalid Rust enum discriminant. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct TracerValueToken<'a> { + pub kind: u8, + pub bool_value: u8, + pub child_count: u32, + pub i64_value: i64, + pub u64_value: u64, + pub f64_value: f64, + pub bytes: ByteSlice<'a>, +} + +/// Opaque owned MessagePack blob produced from structured-value tokens. +pub struct TracerEncodedValue(Vec); + +fn invalid_input(message: &str) -> Box { + Box::new(ExporterError::new(ErrorCode::InvalidInput, message)) +} + +fn invalid_argument(message: &str) -> Box { + Box::new(ExporterError::new(ErrorCode::InvalidArgument, message)) +} + +/// Adapts an `rmp` write failure into an exporter error. Writing into a `Vec` +/// cannot actually fail, so this only exists to satisfy the fallible `rmp` API. +fn encoding_failed(_: E) -> Box { + Box::new(ExporterError::new( + ErrorCode::Internal, + "structured value encoding failed", + )) +} + +/// `rmp`'s string and binary writers narrow the length to `u32` internally, so +/// reject anything longer up front rather than emit a truncated length prefix. +fn ensure_byte_len_fits(bytes: &[u8]) -> Result<(), Box> { + u32::try_from(bytes.len()) + .map(|_| ()) + .map_err(|_| invalid_input("structured value byte string exceeds u32::MAX")) +} + +fn encode_one( + tokens: &[TracerValueToken<'_>], + index: &mut usize, + depth: u32, + output: &mut Vec, +) -> Result<(), Box> { + let token = tokens + .get(*index) + .ok_or_else(|| invalid_input("structured value container is missing child tokens"))?; + *index += 1; + + match token.kind { + kind if kind == TracerValueKind::Nil as u8 => write_nil(output).map_err(encoding_failed)?, + kind if kind == TracerValueKind::Bool as u8 => { + let value = match token.bool_value { + 0 => false, + 1 => true, + _ => return Err(invalid_input("structured value boolean must be 0 or 1")), + }; + write_bool(output, value).map_err(encoding_failed)?; + } + kind if kind == TracerValueKind::I64 as u8 => { + write_sint(output, token.i64_value).map_err(encoding_failed)?; + } + kind if kind == TracerValueKind::U64 as u8 => { + write_uint(output, token.u64_value).map_err(encoding_failed)?; + } + kind if kind == TracerValueKind::F64 as u8 => { + write_f64(output, token.f64_value).map_err(encoding_failed)? + } + kind if kind == TracerValueKind::String as u8 || kind == TracerValueKind::Binary as u8 => { + let bytes = token + .bytes + .try_as_bytes() + .map_err(|_| invalid_argument("structured value contains an invalid byte slice"))?; + ensure_byte_len_fits(bytes)?; + if token.kind == TracerValueKind::String as u8 { + let text = std::str::from_utf8(bytes) + .map_err(|_| invalid_input("structured value string is not valid UTF-8"))?; + write_str(output, text).map_err(encoding_failed)?; + } else { + write_bin(output, bytes).map_err(encoding_failed)?; + } + } + kind if kind == TracerValueKind::Array as u8 || kind == TracerValueKind::Map as u8 => { + if depth >= MAX_DEPTH { + return Err(invalid_input( + "structured value exceeds maximum depth of 64", + )); + } + let values = if token.kind == TracerValueKind::Map as u8 { + write_map_len(output, token.child_count).map_err(encoding_failed)?; + token + .child_count + .checked_mul(2) + .ok_or_else(|| invalid_input("structured value map child count overflows"))? + } else { + write_array_len(output, token.child_count).map_err(encoding_failed)?; + token.child_count + }; + for _ in 0..values { + encode_one(tokens, index, depth + 1, output)?; + } + } + _ => { + return Err(invalid_input( + "structured value contains an unknown token kind", + )) + } + } + Ok(()) +} + +/// Encode one flat preorder structured value as an owned MessagePack blob. +/// +/// On success, `out_handle` receives an owned blob that must be freed with +/// [`ddog_tracer_encoded_value_free`]. The input is fully validated and must +/// contain exactly one value. Token byte slices are borrowed only for this +/// synchronous call; the returned blob does not retain them. +/// +/// # Safety +/// +/// `tokens` and every byte slice referenced by its tokens must remain valid for +/// this call. `out_handle` must point to writable memory for a +/// `Box`. +#[no_mangle] +pub unsafe extern "C" fn ddog_tracer_encode_value( + tokens: Slice>, + out_handle: NonNull>, +) -> Option> { + catch_panic!( + { + let inner = || -> Result<(), Box> { + let tokens = tokens + .try_as_slice() + .map_err(|_| invalid_argument("structured value token slice is invalid"))?; + if tokens.is_empty() { + return Err(invalid_input("structured value token slice is empty")); + } + let mut output = Vec::new(); + let mut index = 0; + encode_one(tokens, &mut index, 0, &mut output)?; + if index != tokens.len() { + return Err(invalid_input("structured value has trailing tokens")); + } + out_handle + .as_ptr() + .write(Box::new(TracerEncodedValue(output))); + Ok(()) + }; + inner().err() + }, + gen_error!(ErrorCode::Panic) + ) +} + +/// Borrow the bytes in an encoded value. The slice is valid until the blob is +/// freed. +#[no_mangle] +pub extern "C" fn ddog_tracer_encoded_value_as_slice( + value: Option<&TracerEncodedValue>, +) -> ByteSlice<'_> { + value + .map(|value| ByteSlice::from(value.0.as_slice())) + .unwrap_or_default() +} + +/// Free an encoded structured value and its bytes. +#[no_mangle] +pub extern "C" fn ddog_tracer_encoded_value_free(value: Option>) { + drop(value); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::MaybeUninit; + + fn token<'a>(kind: u8) -> TracerValueToken<'a> { + TracerValueToken { + kind, + bool_value: 0, + child_count: 0, + i64_value: 0, + u64_value: 0, + f64_value: 0.0, + bytes: ByteSlice::empty(), + } + } + + unsafe fn encode(tokens: &[TracerValueToken<'_>]) -> Result, Box> { + let blob = encode_blob(tokens)?; + let bytes = ddog_tracer_encoded_value_as_slice(Some(&blob)) + .as_bytes() + .to_vec(); + ddog_tracer_encoded_value_free(Some(blob)); + Ok(bytes) + } + + unsafe fn encode_blob( + tokens: &[TracerValueToken<'_>], + ) -> Result, Box> { + let mut handle = MaybeUninit::>::uninit(); + let out = NonNull::new(handle.as_mut_ptr()).unwrap(); + if let Some(error) = ddog_tracer_encode_value(Slice::from(tokens), out) { + return Err(error); + } + Ok(handle.assume_init()) + } + + #[test] + fn encodes_all_supported_values() { + let mut map = token(DDOG_TRACER_VALUE_MAP); + map.child_count = 7; + let mut bools = token(DDOG_TRACER_VALUE_ARRAY); + bools.child_count = 2; + let mut false_token = token(DDOG_TRACER_VALUE_BOOL); + false_token.bool_value = 0; + let mut true_token = token(DDOG_TRACER_VALUE_BOOL); + true_token.bool_value = 1; + let mut signed = token(DDOG_TRACER_VALUE_I64); + signed.i64_value = i64::MIN; + let mut unsigned = token(DDOG_TRACER_VALUE_U64); + unsigned.u64_value = u64::MAX; + let mut float = token(DDOG_TRACER_VALUE_F64); + float.f64_value = 1.25; + let mut binary = token(DDOG_TRACER_VALUE_BINARY); + binary.bytes = ByteSlice::from(&b"\0\xff"[..]); + + let strings = [ + "nil", "bools", "signed", "unsigned", "float", "string", "binary", "hello", + ]; + let string_tokens: Vec<_> = strings + .iter() + .map(|value| { + let mut t = token(DDOG_TRACER_VALUE_STRING); + t.bytes = ByteSlice::from(value.as_bytes()); + t + }) + .collect(); + let tokens = [ + map, + string_tokens[0], + token(DDOG_TRACER_VALUE_NIL), + string_tokens[1], + bools, + false_token, + true_token, + string_tokens[2], + signed, + string_tokens[3], + unsigned, + string_tokens[4], + float, + string_tokens[5], + string_tokens[7], + string_tokens[6], + binary, + ]; + + let encoded = unsafe { encode(&tokens).unwrap() }; + let mut expected = vec![ + 0x87, 0xa3, b'n', b'i', b'l', 0xc0, 0xa5, b'b', b'o', b'o', b'l', b's', 0x92, 0xc2, + 0xc3, 0xa6, b's', b'i', b'g', b'n', b'e', b'd', 0xd3, + ]; + expected.extend_from_slice(&i64::MIN.to_be_bytes()); + expected.extend_from_slice(&[0xa8, b'u', b'n', b's', b'i', b'g', b'n', b'e', b'd', 0xcf]); + expected.extend_from_slice(&u64::MAX.to_be_bytes()); + expected.extend_from_slice(&[0xa5, b'f', b'l', b'o', b'a', b't', 0xcb]); + expected.extend_from_slice(&1.25f64.to_be_bytes()); + expected.extend_from_slice(&[ + 0xa6, b's', b't', b'r', b'i', b'n', b'g', 0xa5, b'h', b'e', b'l', b'l', b'o', 0xa6, + b'b', b'i', b'n', b'a', b'r', b'y', 0xc4, 0x02, 0x00, 0xff, + ]); + assert_eq!(encoded, expected); + } + + #[test] + fn rejects_empty_missing_and_trailing_tokens() { + assert!(unsafe { encode(&[]) }.is_err()); + + let mut array = token(DDOG_TRACER_VALUE_ARRAY); + array.child_count = 1; + assert!(unsafe { encode(&[array]) }.is_err()); + assert!( + unsafe { encode(&[token(DDOG_TRACER_VALUE_NIL), token(DDOG_TRACER_VALUE_NIL),]) } + .is_err() + ); + } + + #[test] + fn rejects_invalid_kinds_booleans_and_utf8() { + assert!(unsafe { encode(&[token(255)]) }.is_err()); + + let mut boolean = token(DDOG_TRACER_VALUE_BOOL); + boolean.bool_value = 2; + assert!(unsafe { encode(&[boolean]) }.is_err()); + + let mut string = token(DDOG_TRACER_VALUE_STRING); + string.bytes = ByteSlice::from(&b"\xff"[..]); + assert!(unsafe { encode(&[string]) }.is_err()); + } + + #[test] + fn rejects_excessive_depth_and_map_count_overflow() { + let mut array = token(DDOG_TRACER_VALUE_ARRAY); + array.child_count = 1; + let mut tokens = vec![array; MAX_DEPTH as usize + 1]; + tokens.push(token(DDOG_TRACER_VALUE_NIL)); + assert!(unsafe { encode(&tokens) }.is_err()); + + let mut map = token(DDOG_TRACER_VALUE_MAP); + map.child_count = u32::MAX; + assert!(unsafe { encode(&[map]) }.is_err()); + } + + #[test] + fn encodes_length_boundaries() { + for (kind, marker32, marker255, marker256, marker65536) in [ + (DDOG_TRACER_VALUE_STRING, 0xd9, 0xd9, 0xda, 0xdb), + (DDOG_TRACER_VALUE_BINARY, 0xc4, 0xc4, 0xc5, 0xc6), + ] { + for (len, expected) in [ + (32, vec![marker32, 32]), + (255, vec![marker255, 255]), + (256, vec![marker256, 1, 0]), + (65_536, vec![marker65536, 0, 1, 0, 0]), + ] { + let bytes = vec![b'a'; len]; + let mut value = token(kind); + value.bytes = ByteSlice::from(bytes.as_slice()); + let encoded = unsafe { encode(&[value]).unwrap() }; + assert_eq!(&encoded[..expected.len()], expected); + assert_eq!(encoded.len(), expected.len() + len); + } + } + + let bytes31 = [b'a'; 31]; + let mut string31 = token(DDOG_TRACER_VALUE_STRING); + string31.bytes = ByteSlice::from(&bytes31[..]); + assert_eq!(unsafe { encode(&[string31]).unwrap() }[0], 0xbf); + + for (kind, count, header, values_per_entry) in [ + (DDOG_TRACER_VALUE_ARRAY, 15, vec![0x9f], 1), + (DDOG_TRACER_VALUE_ARRAY, 16, vec![0xdc, 0, 16], 1), + (DDOG_TRACER_VALUE_MAP, 15, vec![0x8f], 2), + (DDOG_TRACER_VALUE_MAP, 16, vec![0xde, 0, 16], 2), + ] { + let mut container = token(kind); + container.child_count = count; + let mut tokens = vec![container]; + tokens.extend(std::iter::repeat_n( + token(DDOG_TRACER_VALUE_NIL), + count as usize * values_per_entry, + )); + let encoded = unsafe { encode(&tokens).unwrap() }; + assert_eq!(&encoded[..header.len()], header); + } + } + + #[test] + fn rejects_invalid_token_slice() { + let tokens = unsafe { Slice::from_raw_parts(std::ptr::null(), 1) }; + let mut handle = MaybeUninit::>::uninit(); + let out = NonNull::new(handle.as_mut_ptr()).unwrap(); + let error = unsafe { ddog_tracer_encode_value(tokens, out) }; + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + } + + #[test] + fn null_blob_access_is_empty_and_free_is_safe() { + assert!(ddog_tracer_encoded_value_as_slice(None).is_empty()); + ddog_tracer_encoded_value_free(None); + } + + #[test] + fn returned_blob_does_not_borrow_token_bytes() { + let mut backing = b"stable".to_vec(); + let blob = { + let mut string = token(DDOG_TRACER_VALUE_STRING); + string.bytes = ByteSlice::from(backing.as_slice()); + unsafe { encode_blob(&[string]).unwrap() } + }; + + backing.fill(b'x'); + assert_eq!( + ddog_tracer_encoded_value_as_slice(Some(&blob)).as_bytes(), + b"\xa6stable" + ); + ddog_tracer_encoded_value_free(Some(blob)); + } +}