From 7f8d9db0591aa615638ec74dbb98891ca5c54d7f Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 16:11:30 -0400 Subject: [PATCH 01/68] feat(opentelemetry sink): add gRPC transport support --- Cargo.toml | 2 +- .../opentelemetry_sink_grpc.feature.md | 1 + src/sinks/opentelemetry/grpc.rs | 667 ++++++++++++++++++ src/sinks/opentelemetry/mod.rs | 21 +- 4 files changed, 685 insertions(+), 6 deletions(-) create mode 100644 changelog.d/opentelemetry_sink_grpc.feature.md create mode 100644 src/sinks/opentelemetry/grpc.rs diff --git a/Cargo.toml b/Cargo.toml index 1cfdf70ae5530..3c7000d570900 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -909,7 +909,7 @@ sinks-mqtt = ["dep:rumqttc"] sinks-nats = ["dep:async-nats", "dep:nkeys"] sinks-new_relic_logs = ["sinks-http"] sinks-new_relic = [] -sinks-opentelemetry = ["sinks-http", "codecs-opentelemetry"] +sinks-opentelemetry = ["sinks-http", "codecs-opentelemetry", "dep:tonic", "dep:prost"] sinks-papertrail = ["dep:syslog"] sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"] sinks-postgres = ["dep:sqlx"] diff --git a/changelog.d/opentelemetry_sink_grpc.feature.md b/changelog.d/opentelemetry_sink_grpc.feature.md new file mode 100644 index 0000000000000..6c5fa5da6caa0 --- /dev/null +++ b/changelog.d/opentelemetry_sink_grpc.feature.md @@ -0,0 +1 @@ +Added gRPC transport support for the `opentelemetry` sink. Configure with `protocol.type = "grpc"` and set `protocol.endpoint` to your OTLP/gRPC endpoint (e.g. `http://localhost:4317`). Supports TLS and gzip compression. diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs new file mode 100644 index 0000000000000..b84708fa91fa5 --- /dev/null +++ b/src/sinks/opentelemetry/grpc.rs @@ -0,0 +1,667 @@ +use std::{ + fmt, + num::NonZeroUsize, + task::{Context, Poll}, +}; + +use async_trait::async_trait; +use bytes::BytesMut; +use futures::{StreamExt, TryFutureExt, future::BoxFuture, stream::BoxStream}; +use http::Uri; +use hyper::client::HttpConnector; +use hyper_openssl::HttpsConnector; +use hyper_proxy::ProxyConnector; +use prost::Message; +use snafu::Snafu; +use tokio_util::codec::Encoder; +use tonic::body::BoxBody; +use tower::{Service, ServiceBuilder}; +use vector_lib::{ + ByteSizeOf, EstimatedJsonEncodedSizeOf, + codecs::encoding::format::OtlpSerializer, + config::telemetry, + configurable::configurable_component, + opentelemetry::proto::{ + RESOURCE_LOGS_JSON_FIELD, RESOURCE_METRICS_JSON_FIELD, RESOURCE_SPANS_JSON_FIELD, + collector::{ + logs::v1::{ExportLogsServiceRequest, logs_service_client::LogsServiceClient}, + metrics::v1::{ + ExportMetricsServiceRequest, metrics_service_client::MetricsServiceClient, + }, + trace::v1::{ExportTraceServiceRequest, trace_service_client::TraceServiceClient}, + }, + }, + request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, + stream::{BatcherSettings, DriverResponse, batcher::data::BatchReduce}, +}; + +use crate::{ + config::{AcknowledgementsConfig, Input, SinkContext}, + event::{Event, EventFinalizers, EventStatus, Finalizable}, + http::build_proxy_connector, + internal_events::EndpointBytesSent, + sinks::{ + Healthcheck, VectorSink, + util::{ + BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, + StreamSink, TowerRequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, + }, + }, + tls::{MaybeTlsSettings, TlsEnableableConfig}, +}; + +pub(super) fn with_default_scheme(address: &str, tls: bool) -> crate::Result { + let uri: Uri = address.parse()?; + if uri.scheme().is_none() { + let mut parts = uri.into_parts(); + parts.scheme = if tls { + Some( + "https" + .parse() + .unwrap_or_else(|_| unreachable!("https should be valid")), + ) + } else { + Some( + "http" + .parse() + .unwrap_or_else(|_| unreachable!("http should be valid")), + ) + }; + if parts.path_and_query.is_none() { + parts.path_and_query = Some( + "/".parse() + .unwrap_or_else(|_| unreachable!("root should be valid")), + ); + } + Ok(Uri::from_parts(parts)?) + } else { + Ok(uri) + } +} + +/// Configuration for the OpenTelemetry sink's gRPC transport. +#[configurable_component] +#[derive(Clone, Debug)] +pub struct GrpcSinkConfig { + /// The endpoint to send gRPC requests to. + /// + /// The endpoint _must_ include a port. If scheme is omitted, `http` or `https` is + /// inferred from the TLS configuration. + /// + /// # Examples + /// + /// - `http://localhost:4317` + /// - `https://otelcol.example.com:4317` + #[configurable(validation(format = "uri"))] + #[configurable(metadata(docs::examples = "http://localhost:4317"))] + pub endpoint: String, + + /// Whether to compress outgoing requests with gzip. + /// + /// Defaults to `false`. + #[serde(default)] + #[configurable(metadata(docs::advanced))] + pub compression: bool, + + #[configurable(derived)] + #[serde(default)] + pub batch: BatchConfig, + + #[configurable(derived)] + #[serde(default)] + pub request: TowerRequestConfig, + + #[configurable(derived)] + #[serde(default)] + pub tls: Option, + + #[configurable(derived)] + #[serde( + default, + deserialize_with = "crate::serde::bool_or_struct", + skip_serializing_if = "crate::serde::is_default" + )] + pub acknowledgements: AcknowledgementsConfig, +} + +impl GrpcSinkConfig { + pub async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), false)?; + let uri = with_default_scheme(&self.endpoint, tls.is_tls())?; + + let client = new_grpc_client(&tls, cx.proxy())?; + let service = OtlpGrpcService::new(client, uri, self.compression); + + let healthcheck = Box::pin(async move { Ok(()) }); + + let request_settings = self.request.into_settings(); + let batch_settings = self.batch.into_batcher_settings()?; + + let service = ServiceBuilder::new() + .settings(request_settings, OtlpGrpcRetryLogic) + .service(service); + + let sink = OtlpGrpcSink { + batch_settings, + service, + }; + + Ok((VectorSink::from_event_streamsink(sink), healthcheck)) + } + + pub fn input(&self) -> Input { + Input::all() + } + + pub const fn acknowledgements(&self) -> &AcknowledgementsConfig { + &self.acknowledgements + } +} + +fn new_grpc_client( + tls_settings: &MaybeTlsSettings, + proxy_config: &crate::config::ProxyConfig, +) -> crate::Result>, BoxBody>> { + let proxy = build_proxy_connector(tls_settings.clone(), proxy_config)?; + Ok(hyper::Client::builder().http2_only(true).build(proxy)) +} + +// ── Retry logic ────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +struct OtlpGrpcRetryLogic; + +impl RetryLogic for OtlpGrpcRetryLogic { + type Error = OtlpGrpcError; + type Request = OtlpGrpcRequest; + type Response = OtlpGrpcResponse; + + fn is_retriable_error(&self, err: &Self::Error) -> bool { + use tonic::Code::*; + + match err { + OtlpGrpcError::Request { source } => !matches!( + source.code(), + // List taken from + // + NotFound + | InvalidArgument + | AlreadyExists + | PermissionDenied + | OutOfRange + | Unimplemented + | Unauthenticated + | DataLoss + ), + } + } +} + +// ── Error type ─────────────────────────────────────────────────────────────── + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum OtlpGrpcError { + #[snafu(display("gRPC request failed: {source}"))] + Request { source: tonic::Status }, +} + +// ── Request/Response ───────────────────────────────────────────────────────── + +/// A grouped batch of OTLP events, with one proto request per signal type. +#[derive(Clone, Default)] +pub struct OtlpGrpcRequest { + pub logs: Option, + pub metrics: Option, + pub traces: Option, + pub finalizers: EventFinalizers, + pub metadata: RequestMetadata, +} + +impl Finalizable for OtlpGrpcRequest { + fn take_finalizers(&mut self) -> EventFinalizers { + self.finalizers.take_finalizers() + } +} + +impl MetaDescriptive for OtlpGrpcRequest { + fn get_metadata(&self) -> &RequestMetadata { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut RequestMetadata { + &mut self.metadata + } +} + +pub struct OtlpGrpcResponse { + events_byte_size: GroupedCountByteSize, +} + +impl DriverResponse for OtlpGrpcResponse { + fn event_status(&self) -> EventStatus { + EventStatus::Delivered + } + + fn events_sent(&self) -> &GroupedCountByteSize { + &self.events_byte_size + } +} + +// ── Service ────────────────────────────────────────────────────────────────── + +#[derive(Clone)] +pub struct OtlpGrpcService { + logs_client: LogsServiceClient, + metrics_client: MetricsServiceClient, + traces_client: TraceServiceClient, + protocol: String, + endpoint: String, +} + +impl OtlpGrpcService { + pub fn new( + hyper_client: hyper::Client>, BoxBody>, + uri: Uri, + compression: bool, + ) -> Self { + let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(uri.clone()); + + let svc = HyperSvc { + uri, + client: hyper_client, + }; + + let mut logs_client = LogsServiceClient::new(svc.clone()); + let mut metrics_client = MetricsServiceClient::new(svc.clone()); + let mut traces_client = TraceServiceClient::new(svc); + + if compression { + logs_client = logs_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); + metrics_client = + metrics_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); + traces_client = traces_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); + } + + Self { + logs_client, + metrics_client, + traces_client, + protocol, + endpoint, + } + } +} + +impl Service for OtlpGrpcService { + type Response = OtlpGrpcResponse; + type Error = crate::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, mut req: OtlpGrpcRequest) -> Self::Future { + let mut svc = self.clone(); + let metadata = std::mem::take(req.metadata_mut()); + let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); + + let future = async move { + let mut total_bytes: usize = 0; + + if let Some(logs_req) = req.logs { + total_bytes += logs_req.encoded_len(); + svc.logs_client + .export(logs_req) + .map_err(|source| OtlpGrpcError::Request { source }) + .await?; + } + + if let Some(metrics_req) = req.metrics { + total_bytes += metrics_req.encoded_len(); + svc.metrics_client + .export(metrics_req) + .map_err(|source| OtlpGrpcError::Request { source }) + .await?; + } + + if let Some(traces_req) = req.traces { + total_bytes += traces_req.encoded_len(); + svc.traces_client + .export(traces_req) + .map_err(|source| OtlpGrpcError::Request { source }) + .await?; + } + + emit!(EndpointBytesSent { + byte_size: total_bytes, + protocol: &svc.protocol, + endpoint: &svc.endpoint, + }); + + Ok(OtlpGrpcResponse { events_byte_size }) + }; + + Box::pin(future.map_err(|err: OtlpGrpcError| -> crate::Error { Box::new(err) })) + } +} + +// ── HyperSvc (same as in sinks/vector/service.rs) ──────────────────────────── + +#[derive(Clone)] +pub struct HyperSvc { + uri: Uri, + client: hyper::Client>, BoxBody>, +} + +impl Service> for HyperSvc { + type Response = hyper::Response; + type Error = hyper::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, mut req: hyper::Request) -> Self::Future { + let uri = Uri::builder() + .scheme(self.uri.scheme().unwrap().clone()) + .authority(self.uri.authority().unwrap().clone()) + .path_and_query(req.uri().path_and_query().unwrap().clone()) + .build() + .unwrap(); + + *req.uri_mut() = uri; + + Box::pin(self.client.request(req)) + } +} + +// ── Sink ───────────────────────────────────────────────────────────────────── + +/// Intermediate event data extracted before batching. +struct OtlpEventData { + byte_size: usize, + json_byte_size: GroupedCountByteSize, + finalizers: EventFinalizers, + signal: OtlpSignal, +} + +/// Pre-decoded OTLP signal for a single event. +enum OtlpSignal { + Logs(ExportLogsServiceRequest), + Metrics(ExportMetricsServiceRequest), + Traces(ExportTraceServiceRequest), +} + +impl OtlpSignal { + fn encoded_len(&self) -> usize { + match self { + OtlpSignal::Logs(r) => r.encoded_len(), + OtlpSignal::Metrics(r) => r.encoded_len(), + OtlpSignal::Traces(r) => r.encoded_len(), + } + } +} + +/// Accumulator for a batch of OTLP events, merged per signal type. +#[derive(Default)] +struct OtlpBatch { + finalizers: EventFinalizers, + event_count: usize, + events_byte_size: usize, + events_json_byte_size: GroupedCountByteSize, + logs: Option, + metrics: Option, + traces: Option, +} + +impl Clone for OtlpBatch { + fn clone(&self) -> Self { + // OtlpBatch is used only in the batcher accumulator; Clone is required by the + // BatchReduce API but the accumulator is always the initial default value. + Self::default() + } +} + +pub struct OtlpGrpcSink { + pub batch_settings: BatcherSettings, + pub service: S, +} + +impl OtlpGrpcSink +where + S: Service + Send + 'static, + S::Future: Send + 'static, + S::Response: DriverResponse + Send + 'static, + S::Error: fmt::Debug + Into + Send, +{ + async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + let mut serializer = OtlpSerializer::new().map_err(|e| { + error!("Failed to create OTLP serializer: {}", e); + })?; + + input + .filter_map(|mut event| { + let signal = encode_event(&mut serializer, &event); + match signal { + Ok(Some(signal)) => { + let mut json_byte_size = telemetry().create_request_count_byte_size(); + json_byte_size.add_event(&event, event.estimated_json_encoded_size_of()); + let data = OtlpEventData { + byte_size: event.size_of(), + json_byte_size, + finalizers: event.take_finalizers(), + signal, + }; + futures::future::ready(Some(data)) + } + Ok(None) => { + // Event type not supported (e.g. native Vector Metric) + emit!(crate::internal_events::SinkRequestBuildError { + error: "Unsupported event type for OTLP gRPC sink (native Vector Metric events are not supported; use OTLP-decoded metrics)", + }); + futures::future::ready(None) + } + Err(e) => { + emit!(crate::internal_events::SinkRequestBuildError { error: e }); + futures::future::ready(None) + } + } + }) + .batched(self.batch_settings.as_reducer_config( + |data: &OtlpEventData| data.signal.encoded_len(), + BatchReduce::new(|batch: &mut OtlpBatch, item: OtlpEventData| { + batch.finalizers.merge(item.finalizers); + batch.event_count += 1; + batch.events_byte_size += item.byte_size; + batch.events_json_byte_size += item.json_byte_size; + match item.signal { + OtlpSignal::Logs(req) => { + if let Some(existing) = &mut batch.logs { + existing.resource_logs.extend(req.resource_logs); + } else { + batch.logs = Some(req); + } + } + OtlpSignal::Metrics(req) => { + if let Some(existing) = &mut batch.metrics { + existing.resource_metrics.extend(req.resource_metrics); + } else { + batch.metrics = Some(req); + } + } + OtlpSignal::Traces(req) => { + if let Some(existing) = &mut batch.traces { + existing.resource_spans.extend(req.resource_spans); + } else { + batch.traces = Some(req); + } + } + } + }), + )) + .map(|batch| { + let builder = RequestMetadataBuilder::new( + batch.event_count, + batch.events_byte_size, + batch.events_json_byte_size, + ); + + let byte_size = batch + .logs + .as_ref() + .map_or(0, |r| r.encoded_len()) + + batch + .metrics + .as_ref() + .map_or(0, |r| r.encoded_len()) + + batch + .traces + .as_ref() + .map_or(0, |r| r.encoded_len()); + + let bytes_len = + NonZeroUsize::new(byte_size.max(1)).expect("should be non-zero"); + + OtlpGrpcRequest { + logs: batch.logs, + metrics: batch.metrics, + traces: batch.traces, + finalizers: batch.finalizers, + metadata: builder.with_request_size(bytes_len), + } + }) + .into_driver(self.service) + .run() + .await + } +} + +#[async_trait] +impl StreamSink for OtlpGrpcSink +where + S: Service + Send + 'static, + S::Future: Send + 'static, + S::Response: DriverResponse + Send + 'static, + S::Error: fmt::Debug + Into + Send, +{ + async fn run(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + self.run_inner(input).await + } +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/// Encode a Vector event into an OTLP proto signal using [`OtlpSerializer`]. +/// +/// Returns `Ok(None)` for native Vector `Metric` events (not supported by OTLP +/// serializer). Returns `Err` for unexpected encoding failures. +fn encode_event( + serializer: &mut OtlpSerializer, + event: &Event, +) -> Result, vector_common::Error> { + let signal_type = detect_signal_type(event); + let signal_type = match signal_type { + Some(t) => t, + None => return Ok(None), + }; + + let mut buf = BytesMut::new(); + // Clone the event to pass ownership to the encoder while keeping the original + // for metadata extraction in the caller. The clone is shallow for logs/traces. + serializer.encode(event.clone(), &mut buf)?; + + let bytes = buf.freeze(); + match signal_type { + SignalType::Logs => { + let req = ExportLogsServiceRequest::decode(bytes)?; + Ok(Some(OtlpSignal::Logs(req))) + } + SignalType::Metrics => { + let req = ExportMetricsServiceRequest::decode(bytes)?; + Ok(Some(OtlpSignal::Metrics(req))) + } + SignalType::Traces => { + let req = ExportTraceServiceRequest::decode(bytes)?; + Ok(Some(OtlpSignal::Traces(req))) + } + } +} + +enum SignalType { + Logs, + Metrics, + Traces, +} + +fn detect_signal_type(event: &Event) -> Option { + match event { + Event::Log(log) => { + if log.contains(RESOURCE_LOGS_JSON_FIELD) { + Some(SignalType::Logs) + } else if log.contains(RESOURCE_METRICS_JSON_FIELD) { + Some(SignalType::Metrics) + } else { + None + } + } + Event::Trace(trace) => { + if trace.contains(RESOURCE_SPANS_JSON_FIELD) { + Some(SignalType::Traces) + } else { + None + } + } + Event::Metric(_) => None, // Native Vector metrics not supported by OtlpSerializer + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generate_grpc_config() { + let config: GrpcSinkConfig = toml::from_str( + r#" + endpoint = "http://localhost:4317" + "#, + ) + .unwrap(); + assert_eq!(config.endpoint, "http://localhost:4317"); + assert!(!config.compression); + } + + #[test] + fn grpc_config_with_tls() { + let config: GrpcSinkConfig = toml::from_str( + r#" + endpoint = "https://otelcol.example.com:4317" + compression = true + "#, + ) + .unwrap(); + assert_eq!(config.endpoint, "https://otelcol.example.com:4317"); + assert!(config.compression); + } + + #[test] + fn with_default_scheme_adds_http() { + let uri = with_default_scheme("localhost:4317", false).unwrap(); + assert_eq!(uri.scheme_str(), Some("http")); + } + + #[test] + fn with_default_scheme_adds_https() { + let uri = with_default_scheme("localhost:4317", true).unwrap(); + assert_eq!(uri.scheme_str(), Some("https")); + } + + #[test] + fn with_default_scheme_preserves_existing() { + let uri = with_default_scheme("http://localhost:4317", true).unwrap(); + assert_eq!(uri.scheme_str(), Some("http")); + } +} diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 88963f8603cde..0a8a24a153215 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -1,3 +1,5 @@ +mod grpc; + use indoc::indoc; use vector_config::component::GenerateConfig; use vector_lib::{ @@ -17,8 +19,10 @@ use crate::{ }, }; +pub use grpc::GrpcSinkConfig; + /// Configuration for the `OpenTelemetry` sink. -#[configurable_component(sink("opentelemetry", "Deliver OTLP data over HTTP."))] +#[configurable_component(sink("opentelemetry", "Deliver OTLP data over HTTP or gRPC."))] #[derive(Clone, Debug, Default)] pub struct OpenTelemetryConfig { /// Protocol configuration @@ -26,16 +30,20 @@ pub struct OpenTelemetryConfig { protocol: Protocol, } -/// The protocol used to send data to OpenTelemetry. -/// Currently only HTTP is supported, but we plan to support gRPC. +/// The protocol used to send data to an OpenTelemetry-compatible endpoint. +/// /// The proto definitions are defined [here](https://github.com/vectordotdev/vector/blob/master/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/README.md). #[configurable_component] #[derive(Clone, Debug)] #[serde(rename_all = "snake_case", tag = "type")] #[configurable(metadata(docs::enum_tag_description = "The communication protocol."))] +#[allow(clippy::large_enum_variant)] pub enum Protocol { /// Send data over HTTP. Http(HttpSinkConfig), + + /// Send data over gRPC. + Grpc(GrpcSinkConfig), } impl Default for Protocol { @@ -79,18 +87,21 @@ impl SinkConfig for OpenTelemetryConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { match &self.protocol { Protocol::Http(config) => config.build(cx).await, + Protocol::Grpc(config) => config.build(cx).await, } } fn input(&self) -> Input { match &self.protocol { Protocol::Http(config) => config.input(), + Protocol::Grpc(config) => config.input(), } } fn acknowledgements(&self) -> &AcknowledgementsConfig { - match self.protocol { - Protocol::Http(ref config) => config.acknowledgements(), + match &self.protocol { + Protocol::Http(config) => config.acknowledgements(), + Protocol::Grpc(config) => config.acknowledgements(), } } } From f8bc78e39d181b1b21c857cac39fbc46bf90655a Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 16:47:51 -0400 Subject: [PATCH 02/68] test(opentelemetry sink): add gRPC integration and e2e tests --- Cargo.toml | 2 +- src/sinks/opentelemetry/integration_tests.rs | 76 +++++++++++++++++++ src/sinks/opentelemetry/mod.rs | 3 + .../opentelemetry-logs/config/compose.yaml | 1 + tests/e2e/opentelemetry-logs/config/test.yaml | 3 +- .../data/collector-sink.yaml | 2 + .../opentelemetry-logs/data/vector_grpc.yaml | 40 ++++++++++ .../opentelemetry/config/test.yaml | 2 + .../opentelemetry/data/config.yaml | 2 + 9 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 src/sinks/opentelemetry/integration_tests.rs create mode 100644 tests/e2e/opentelemetry-logs/data/vector_grpc.yaml diff --git a/Cargo.toml b/Cargo.toml index 3c7000d570900..ff78f94ac4a01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1030,7 +1030,7 @@ mongodb_metrics-integration-tests = ["sources-mongodb_metrics"] mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"] nats-integration-tests = ["sinks-nats", "sources-nats"] nginx-integration-tests = ["sources-nginx_metrics"] -opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"] +opentelemetry-integration-tests = ["sources-opentelemetry", "sinks-opentelemetry", "dep:prost"] postgresql_metrics-integration-tests = ["sources-postgresql_metrics"] postgres_sink-integration-tests = ["sinks-postgres"] prometheus-integration-tests = ["sinks-prometheus", "sources-prometheus", "sinks-influxdb"] diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs new file mode 100644 index 0000000000000..9ec0ec472d7a2 --- /dev/null +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -0,0 +1,76 @@ +use futures::stream; +use prost::Message as _; +use vector_lib::{ + codecs::decoding::format::{Deserializer as _, OtlpDeserializer}, + config::LogNamespace, + opentelemetry::proto::{ + collector::logs::v1::ExportLogsServiceRequest, + logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, + resource::v1::Resource, + }, +}; + +use super::grpc::GrpcSinkConfig; +use crate::{ + config::SinkContext, + test_util::{ + components::{HTTP_SINK_TAGS, run_and_assert_sink_compliance}, + wait_for_tcp, + }, +}; + +fn sink_grpc_address() -> String { + std::env::var("OTEL_GRPC_SINK_ADDRESS") + .unwrap_or_else(|_| "opentelemetry-collector:4317".to_owned()) +} + +fn otlp_log_event() -> vector_lib::event::Event { + let req = ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: Some(Resource { + attributes: vec![], + dropped_attributes_count: 0, + }), + scope_logs: vec![ScopeLogs { + scope: None, + log_records: vec![LogRecord { + severity_text: "INFO".to_string(), + body: Some(vector_lib::opentelemetry::proto::common::v1::AnyValue { + value: Some( + vector_lib::opentelemetry::proto::common::v1::any_value::Value::StringValue( + "integration test log message".to_string(), + ), + ), + }), + ..Default::default() + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + + let bytes = bytes::Bytes::from(req.encode_to_vec()); + let mut events = OtlpDeserializer::default() + .parse(bytes, LogNamespace::Legacy) + .expect("failed to deserialize OTLP log event"); + events.remove(0) +} + +#[tokio::test] +async fn delivers_logs_via_grpc() { + let address = sink_grpc_address(); + wait_for_tcp(&address).await; + + let config: GrpcSinkConfig = toml::from_str(&format!( + r#" + endpoint = "http://{address}" + "# + )) + .unwrap(); + + let (sink, _healthcheck) = config.build(SinkContext::default()).await.unwrap(); + + let events = vec![otlp_log_event()]; + run_and_assert_sink_compliance(sink, stream::iter(events), &HTTP_SINK_TAGS).await; +} diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 0a8a24a153215..411bb0ffaaeb8 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -1,5 +1,8 @@ mod grpc; +#[cfg(all(test, feature = "opentelemetry-integration-tests"))] +mod integration_tests; + use indoc::indoc; use vector_config::component::GenerateConfig; use vector_lib::{ diff --git a/tests/e2e/opentelemetry-logs/config/compose.yaml b/tests/e2e/opentelemetry-logs/config/compose.yaml index 62b3d48c44aa6..d193ae5d9b467 100644 --- a/tests/e2e/opentelemetry-logs/config/compose.yaml +++ b/tests/e2e/opentelemetry-logs/config/compose.yaml @@ -60,6 +60,7 @@ services: target: /output ports: - "${OTEL_COLLECTOR_SINK_HTTP_PORT:-5318}:5318" + - "${OTEL_COLLECTOR_SINK_GRPC_PORT:-14317}:4317" vector: container_name: vector-otel-logs-e2e diff --git a/tests/e2e/opentelemetry-logs/config/test.yaml b/tests/e2e/opentelemetry-logs/config/test.yaml index 54aa7063e632c..95fd08d50aadc 100644 --- a/tests/e2e/opentelemetry-logs/config/test.yaml +++ b/tests/e2e/opentelemetry-logs/config/test.yaml @@ -11,11 +11,12 @@ runner: OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + OTEL_COLLECTOR_SINK_GRPC_PORT: '14317' matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use collector_version: [ 'latest' ] - vector_config: [ 'vector_default.yaml', 'vector_otlp.yaml' ] + vector_config: [ 'vector_default.yaml', 'vector_otlp.yaml', 'vector_grpc.yaml' ] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/tests/e2e/opentelemetry-logs/data/collector-sink.yaml b/tests/e2e/opentelemetry-logs/data/collector-sink.yaml index e22f62f91d558..fbc6cc87827a3 100644 --- a/tests/e2e/opentelemetry-logs/data/collector-sink.yaml +++ b/tests/e2e/opentelemetry-logs/data/collector-sink.yaml @@ -3,6 +3,8 @@ receivers: protocols: http: endpoint: "0.0.0.0:5318" + grpc: + endpoint: "0.0.0.0:4317" processors: batch: { } diff --git a/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml b/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml new file mode 100644 index 0000000000000..5ed56ea74b008 --- /dev/null +++ b/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml @@ -0,0 +1,40 @@ +sources: + source0: + type: opentelemetry + grpc: + address: 0.0.0.0:4317 + http: + address: 0.0.0.0:4318 + keepalive: + max_connection_age_jitter_factor: 0.1 + max_connection_age_secs: 300 + use_otlp_decoding: true + + internal_metrics: + type: internal_metrics + scrape_interval_secs: 60 + +sinks: + otel_sink: + inputs: + - source0.logs + type: opentelemetry + protocol: + type: grpc + endpoint: http://otel-collector-sink:4317 + + otel_file_sink: + type: file + path: "/output/opentelemetry-logs/vector-file-sink.log" + inputs: + - source0.logs + encoding: + codec: json + + metrics_file_sink: + type: file + path: "/output/opentelemetry-logs/vector-internal-metrics-sink.log" + inputs: + - internal_metrics + encoding: + codec: json diff --git a/tests/integration/opentelemetry/config/test.yaml b/tests/integration/opentelemetry/config/test.yaml index 91a2968fdfae0..90d058a281a12 100644 --- a/tests/integration/opentelemetry/config/test.yaml +++ b/tests/integration/opentelemetry/config/test.yaml @@ -7,6 +7,7 @@ runner: env: OTEL_HEALTH_URL: http://opentelemetry-collector:13133 OTEL_OTLPHTTP_URL: http://opentelemetry-collector:9876 + OTEL_GRPC_SINK_ADDRESS: opentelemetry-collector:4317 matrix: version: [0.56.0] @@ -15,5 +16,6 @@ matrix: # expressions are evaluated using https://github.com/micromatch/picomatch paths: - "src/sources/opentelemetry/**" +- "src/sinks/opentelemetry/**" - "src/sources/util/**" - "tests/integration/opentelemetry/**" diff --git a/tests/integration/opentelemetry/data/config.yaml b/tests/integration/opentelemetry/data/config.yaml index 66c8fe34e6112..2a0c9ee30c98c 100644 --- a/tests/integration/opentelemetry/data/config.yaml +++ b/tests/integration/opentelemetry/data/config.yaml @@ -3,6 +3,8 @@ receivers: protocols: http: endpoint: 0.0.0.0:9876 + grpc: + endpoint: 0.0.0.0:4317 exporters: otlp: From a0c8e351916e426e587a12ec7007720f071045b4 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 16:56:27 -0400 Subject: [PATCH 03/68] fix(opentelemetry sink): fix lifetime error in integration test --- src/sinks/opentelemetry/integration_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs index 9ec0ec472d7a2..3f17d5b05b99b 100644 --- a/src/sinks/opentelemetry/integration_tests.rs +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -60,7 +60,7 @@ fn otlp_log_event() -> vector_lib::event::Event { #[tokio::test] async fn delivers_logs_via_grpc() { let address = sink_grpc_address(); - wait_for_tcp(&address).await; + wait_for_tcp(address.clone()).await; let config: GrpcSinkConfig = toml::from_str(&format!( r#" From 6526e419990b7c214bd77bb8912b12c80f1de53d Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 18:37:31 -0400 Subject: [PATCH 04/68] fix(opentelemetry sink): align Protocol enum field types to fix schema conflicts --- src/sinks/opentelemetry/grpc.rs | 50 +++++++++++------ .../sinks/generated/opentelemetry.cue | 55 +++++++++++++++---- 2 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index b84708fa91fa5..159fb683668c0 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -43,11 +43,12 @@ use crate::{ sinks::{ Healthcheck, VectorSink, util::{ - BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, - StreamSink, TowerRequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, + BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, + SinkBuilderExt, StreamSink, http::RequestConfig, metadata::RequestMetadataBuilder, + retries::RetryLogic, }, }, - tls::{MaybeTlsSettings, TlsEnableableConfig}, + tls::{MaybeTlsSettings, TlsConfig}, }; pub(super) fn with_default_scheme(address: &str, tls: bool) -> crate::Result { @@ -96,12 +97,12 @@ pub struct GrpcSinkConfig { #[configurable(metadata(docs::examples = "http://localhost:4317"))] pub endpoint: String, - /// Whether to compress outgoing requests with gzip. + /// Compression codec for outgoing gRPC requests. /// - /// Defaults to `false`. + /// Only `none` and `gzip` are supported for gRPC transport. + #[configurable(derived)] #[serde(default)] - #[configurable(metadata(docs::advanced))] - pub compression: bool, + pub compression: Compression, #[configurable(derived)] #[serde(default)] @@ -109,11 +110,11 @@ pub struct GrpcSinkConfig { #[configurable(derived)] #[serde(default)] - pub request: TowerRequestConfig, + pub request: RequestConfig, #[configurable(derived)] #[serde(default)] - pub tls: Option, + pub tls: Option, #[configurable(derived)] #[serde( @@ -126,15 +127,30 @@ pub struct GrpcSinkConfig { impl GrpcSinkConfig { pub async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { - let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), false)?; - let uri = with_default_scheme(&self.endpoint, tls.is_tls())?; + // Determine TLS from the endpoint scheme; fall back to https when tls config is present. + let uri = with_default_scheme(&self.endpoint, self.tls.is_some())?; + let tls = if uri.scheme_str() == Some("https") { + MaybeTlsSettings::tls_client(self.tls.as_ref())? + } else { + MaybeTlsSettings::Raw(()) + }; + let use_gzip = match self.compression { + Compression::None => false, + Compression::Gzip(_) => true, + other => { + return Err(format!( + "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" + ) + .into()) + } + }; let client = new_grpc_client(&tls, cx.proxy())?; - let service = OtlpGrpcService::new(client, uri, self.compression); + let service = OtlpGrpcService::new(client, uri, use_gzip); let healthcheck = Box::pin(async move { Ok(()) }); - let request_settings = self.request.into_settings(); + let request_settings = self.request.tower.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; let service = ServiceBuilder::new() @@ -631,20 +647,20 @@ mod tests { ) .unwrap(); assert_eq!(config.endpoint, "http://localhost:4317"); - assert!(!config.compression); + assert_eq!(config.compression, Compression::default()); } #[test] - fn grpc_config_with_tls() { + fn grpc_config_with_gzip() { let config: GrpcSinkConfig = toml::from_str( r#" endpoint = "https://otelcol.example.com:4317" - compression = true + compression = "gzip" "#, ) .unwrap(); assert_eq!(config.endpoint, "https://otelcol.example.com:4317"); - assert!(config.compression); + assert!(matches!(config.compression, Compression::Gzip(_))); } #[test] diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index f48dfd18a61a4..86572b476fb9c 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1,8 +1,12 @@ package metadata generated: components: sinks: opentelemetry: configuration: protocol: { - description: "Protocol configuration" - required: true + description: """ + The protocol used to send data to an OpenTelemetry-compatible endpoint. + + Protocol configuration + """ + required: true type: object: options: { acknowledgements: { description: """ @@ -37,7 +41,8 @@ generated: components: sinks: opentelemetry: configuration: protocol: { HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an HTTP header without any additional encryption beyond what is provided by the transport itself. """ - required: false + relevant_when: "type = \"http\"" + required: false type: object: options: { auth: { description: "The AWS authentication configuration." @@ -286,7 +291,8 @@ generated: components: sinks: opentelemetry: configuration: protocol: { Configures how events are encoded into raw bytes. The selected encoding also determines which input types (logs, metrics, traces) are supported. """ - required: true + relevant_when: "type = \"http\"" + required: true type: object: options: { avro: { description: "Apache Avro-specific encoder options." @@ -715,9 +721,26 @@ generated: components: sinks: opentelemetry: configuration: protocol: { } } } + endpoint: { + description: """ + The endpoint to send gRPC requests to. + + The endpoint _must_ include a port. If scheme is omitted, `http` or `https` is + inferred from the TLS configuration. + + # Examples + + - `http://localhost:4317` + - `https://otelcol.example.com:4317` + """ + relevant_when: "type = \"grpc\"" + required: true + type: string: examples: ["http://localhost:4317"] + } framing: { - description: "Framing configuration." - required: false + description: "Framing configuration." + relevant_when: "type = \"http\"" + required: false type: object: options: { character_delimited: { description: "Options for the character delimited encoder." @@ -787,6 +810,7 @@ generated: components: sinks: opentelemetry: configuration: protocol: { deprecated: true deprecated_message: "This option has been deprecated, use `request.headers` instead." description: "A list of custom headers to add to each request." + relevant_when: "type = \"http\"" required: false type: object: options: "*": { description: "An HTTP request header and it's value." @@ -795,8 +819,9 @@ generated: components: sinks: opentelemetry: configuration: protocol: { } } method: { - description: "The HTTP method to use when making the request." - required: false + description: "The HTTP method to use when making the request." + relevant_when: "type = \"http\"" + required: false type: string: { default: "post" enum: { @@ -819,7 +844,8 @@ generated: components: sinks: opentelemetry: configuration: protocol: { If specified, the `payload_suffix` must also be specified and together they must produce a valid JSON object. """ - required: false + relevant_when: "type = \"http\"" + required: false type: string: { default: "" examples: ["{\"data\":"] @@ -833,7 +859,8 @@ generated: components: sinks: opentelemetry: configuration: protocol: { If specified, the `payload_prefix` must also be specified and together they must produce a valid JSON object. """ - required: false + relevant_when: "type = \"http\"" + required: false type: string: { default: "" examples: ["}"] @@ -1133,7 +1160,10 @@ generated: components: sinks: opentelemetry: configuration: protocol: { type: { description: "The communication protocol." required: true - type: string: enum: http: "Send data over HTTP." + type: string: enum: { + grpc: "Send data over gRPC." + http: "Send data over HTTP." + } } uri: { description: """ @@ -1141,7 +1171,8 @@ generated: components: sinks: opentelemetry: configuration: protocol: { This should include the protocol and host, but can also include the port, path, and any other valid part of a URI. """ - required: true + relevant_when: "type = \"http\"" + required: true type: string: { examples: ["https://10.22.212.22:9000/endpoint"] syntax: "template" From 41ae7a00aa0fd6275b7d8bf9532dabda30ccbfb1 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 18:38:38 -0400 Subject: [PATCH 05/68] Add authors line to changelog --- changelog.d/opentelemetry_sink_grpc.feature.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.d/opentelemetry_sink_grpc.feature.md b/changelog.d/opentelemetry_sink_grpc.feature.md index 6c5fa5da6caa0..b115672f24dfa 100644 --- a/changelog.d/opentelemetry_sink_grpc.feature.md +++ b/changelog.d/opentelemetry_sink_grpc.feature.md @@ -1 +1,3 @@ Added gRPC transport support for the `opentelemetry` sink. Configure with `protocol.type = "grpc"` and set `protocol.endpoint` to your OTLP/gRPC endpoint (e.g. `http://localhost:4317`). Supports TLS and gzip compression. + +authors: thomasqueirozb From b29ccef276969a4674f0fa0591e79879ff0dc697 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 18:49:01 -0400 Subject: [PATCH 06/68] Use protocol.uri --- src/sinks/opentelemetry/grpc.rs | 37 ++++++++++--------- src/sinks/opentelemetry/integration_tests.rs | 2 +- .../opentelemetry-logs/data/vector_grpc.yaml | 2 +- .../sinks/generated/opentelemetry.cue | 19 +--------- 4 files changed, 22 insertions(+), 38 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 159fb683668c0..43b492f6a98e9 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -44,15 +44,14 @@ use crate::{ Healthcheck, VectorSink, util::{ BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, - SinkBuilderExt, StreamSink, http::RequestConfig, metadata::RequestMetadataBuilder, - retries::RetryLogic, + SinkBuilderExt, StreamSink, UriSerde, http::RequestConfig, + metadata::RequestMetadataBuilder, retries::RetryLogic, }, }, tls::{MaybeTlsSettings, TlsConfig}, }; -pub(super) fn with_default_scheme(address: &str, tls: bool) -> crate::Result { - let uri: Uri = address.parse()?; +pub(super) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { if uri.scheme().is_none() { let mut parts = uri.into_parts(); parts.scheme = if tls { @@ -84,18 +83,17 @@ pub(super) fn with_default_scheme(address: &str, tls: bool) -> crate::Result crate::Result<(VectorSink, Healthcheck)> { - // Determine TLS from the endpoint scheme; fall back to https when tls config is present. - let uri = with_default_scheme(&self.endpoint, self.tls.is_some())?; + // Determine TLS from the URI scheme; fall back to https when tls options are present. + let uri = with_default_scheme(self.uri.uri.clone(), self.tls.is_some())?; let tls = if uri.scheme_str() == Some("https") { MaybeTlsSettings::tls_client(self.tls.as_ref())? } else { @@ -642,11 +640,11 @@ mod tests { fn generate_grpc_config() { let config: GrpcSinkConfig = toml::from_str( r#" - endpoint = "http://localhost:4317" + uri = "http://localhost:4317" "#, ) .unwrap(); - assert_eq!(config.endpoint, "http://localhost:4317"); + assert_eq!(config.uri.uri.to_string(), "http://localhost:4317"); assert_eq!(config.compression, Compression::default()); } @@ -654,30 +652,33 @@ mod tests { fn grpc_config_with_gzip() { let config: GrpcSinkConfig = toml::from_str( r#" - endpoint = "https://otelcol.example.com:4317" + uri = "https://otelcol.example.com:4317" compression = "gzip" "#, ) .unwrap(); - assert_eq!(config.endpoint, "https://otelcol.example.com:4317"); + assert_eq!( + config.uri.uri.to_string(), + "https://otelcol.example.com:4317" + ); assert!(matches!(config.compression, Compression::Gzip(_))); } #[test] fn with_default_scheme_adds_http() { - let uri = with_default_scheme("localhost:4317", false).unwrap(); + let uri = with_default_scheme("localhost:4317".parse().unwrap(), false).unwrap(); assert_eq!(uri.scheme_str(), Some("http")); } #[test] fn with_default_scheme_adds_https() { - let uri = with_default_scheme("localhost:4317", true).unwrap(); + let uri = with_default_scheme("localhost:4317".parse().unwrap(), true).unwrap(); assert_eq!(uri.scheme_str(), Some("https")); } #[test] fn with_default_scheme_preserves_existing() { - let uri = with_default_scheme("http://localhost:4317", true).unwrap(); + let uri = with_default_scheme("http://localhost:4317".parse().unwrap(), true).unwrap(); assert_eq!(uri.scheme_str(), Some("http")); } } diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs index 3f17d5b05b99b..e6dbf470f4cad 100644 --- a/src/sinks/opentelemetry/integration_tests.rs +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -64,7 +64,7 @@ async fn delivers_logs_via_grpc() { let config: GrpcSinkConfig = toml::from_str(&format!( r#" - endpoint = "http://{address}" + uri = "http://{address}" "# )) .unwrap(); diff --git a/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml b/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml index 5ed56ea74b008..99c474d4cb888 100644 --- a/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml +++ b/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml @@ -21,7 +21,7 @@ sinks: type: opentelemetry protocol: type: grpc - endpoint: http://otel-collector-sink:4317 + uri: http://otel-collector-sink:4317 otel_file_sink: type: file diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 86572b476fb9c..9a7ce2c4edc85 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -721,22 +721,6 @@ generated: components: sinks: opentelemetry: configuration: protocol: { } } } - endpoint: { - description: """ - The endpoint to send gRPC requests to. - - The endpoint _must_ include a port. If scheme is omitted, `http` or `https` is - inferred from the TLS configuration. - - # Examples - - - `http://localhost:4317` - - `https://otelcol.example.com:4317` - """ - relevant_when: "type = \"grpc\"" - required: true - type: string: examples: ["http://localhost:4317"] - } framing: { description: "Framing configuration." relevant_when: "type = \"http\"" @@ -1171,8 +1155,7 @@ generated: components: sinks: opentelemetry: configuration: protocol: { This should include the protocol and host, but can also include the port, path, and any other valid part of a URI. """ - relevant_when: "type = \"http\"" - required: true + required: true type: string: { examples: ["https://10.22.212.22:9000/endpoint"] syntax: "template" From d6d0cf06c8b248c8f0ae7ab86a4eb684a8a944e2 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 23 Mar 2026 17:26:05 -0400 Subject: [PATCH 07/68] Flatten otel config --- .../opentelemetry_sink_config.breaking.md | 49 + src/sinks/opentelemetry/mod.rs | 188 +- .../data/vector_default.yaml | 25 +- .../opentelemetry-logs/data/vector_grpc.yaml | 5 +- .../opentelemetry-logs/data/vector_otlp.yaml | 9 +- .../sinks/generated/opentelemetry.cue | 2010 ++++++++--------- 6 files changed, 1184 insertions(+), 1102 deletions(-) create mode 100644 changelog.d/opentelemetry_sink_config.breaking.md diff --git a/changelog.d/opentelemetry_sink_config.breaking.md b/changelog.d/opentelemetry_sink_config.breaking.md new file mode 100644 index 0000000000000..33104d98860a7 --- /dev/null +++ b/changelog.d/opentelemetry_sink_config.breaking.md @@ -0,0 +1,49 @@ +Changed the `opentelemetry` sink config fields to remove `protocol.*`. `protocol.type` was replaced +by `protocol` and all fields previously nested under `protocol` now can be placed in the top level +configuration. + +Before: + +``` +sinks: + otel_sink: + inputs: + - in + protocol: + type: http + uri: http://otel-collector-sink:5318/v1/logs + method: post + encoding: + codec: json + framing: + method: newline_delimited + batch: + max_events: 1 + request: + headers: + content-type: application/json +``` + +After: + +``` +sinks: + otel_sink: + inputs: + - in + type: opentelemetry + protocol: http + uri: http://otel-collector-sink:5318/v1/logs + method: post + encoding: + codec: json + framing: + method: newline_delimited + batch: + max_events: 1 + request: + headers: + content-type: application/json +``` + +authors: thomasqueirozb diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 411bb0ffaaeb8..8862ce635363b 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -5,78 +5,112 @@ mod integration_tests; use indoc::indoc; use vector_config::component::GenerateConfig; -use vector_lib::{ - codecs::{ - JsonSerializerConfig, - encoding::{FramingConfig, SerializerConfig}, - }, - configurable::configurable_component, -}; +use vector_lib::configurable::configurable_component; use crate::{ - codecs::{EncodingConfigWithFraming, Transformer}, + codecs::EncodingConfigWithFraming, config::{AcknowledgementsConfig, Input, SinkConfig, SinkContext}, + http::Auth, sinks::{ Healthcheck, VectorSink, http::config::{HttpMethod, HttpSinkConfig}, + util::{ + BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, + RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::RequestConfig, + }, }, + tls::TlsConfig, }; pub use grpc::GrpcSinkConfig; -/// Configuration for the `OpenTelemetry` sink. -#[configurable_component(sink("opentelemetry", "Deliver OTLP data over HTTP or gRPC."))] -#[derive(Clone, Debug, Default)] -pub struct OpenTelemetryConfig { - /// Protocol configuration - #[configurable(derived)] - protocol: Protocol, -} - -/// The protocol used to send data to an OpenTelemetry-compatible endpoint. -/// -/// The proto definitions are defined [here](https://github.com/vectordotdev/vector/blob/master/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/README.md). +/// Transport protocol for the OpenTelemetry sink. #[configurable_component] #[derive(Clone, Debug)] -#[serde(rename_all = "snake_case", tag = "type")] -#[configurable(metadata(docs::enum_tag_description = "The communication protocol."))] +#[serde(tag = "protocol", rename_all = "snake_case")] #[allow(clippy::large_enum_variant)] -pub enum Protocol { - /// Send data over HTTP. - Http(HttpSinkConfig), +#[configurable(metadata(docs::enum_tag_description = "The transport protocol to use."))] +pub enum OtlpProtocol { + /// Send OTLP data over HTTP. + Http { + /// The HTTP method to use. Defaults to `post`. + #[serde(default)] + method: HttpMethod, + + #[configurable(derived)] + auth: Option, - /// Send data over gRPC. - Grpc(GrpcSinkConfig), + /// Encoding configuration. + #[configurable(derived)] + #[serde(flatten)] + encoding: EncodingConfigWithFraming, + + /// A string to prefix the payload with. + #[serde(default)] + payload_prefix: String, + + /// A string to suffix the payload with. + #[serde(default)] + payload_suffix: String, + + #[configurable(derived)] + #[serde(default)] + batch: BatchConfig, + }, + + /// Send OTLP data over gRPC. + Grpc { + #[configurable(derived)] + #[serde(default)] + batch: BatchConfig, + }, } -impl Default for Protocol { - fn default() -> Self { - Protocol::Http(HttpSinkConfig { - encoding: EncodingConfigWithFraming::new( - Some(FramingConfig::NewlineDelimited), - SerializerConfig::Json(JsonSerializerConfig::default()), - Transformer::default(), - ), - uri: Default::default(), - method: HttpMethod::Post, - auth: Default::default(), - headers: Default::default(), - compression: Default::default(), - payload_prefix: Default::default(), - payload_suffix: Default::default(), - batch: Default::default(), - request: Default::default(), - tls: Default::default(), - acknowledgements: Default::default(), - }) - } +/// Configuration for the `opentelemetry` sink. +#[configurable_component(sink("opentelemetry", "Deliver OTLP data over HTTP or gRPC."))] +#[derive(Clone, Debug)] +pub struct OpenTelemetryConfig { + /// The transport protocol to use. Defaults to `http`. + #[configurable(derived)] + #[serde(flatten)] + pub protocol: OtlpProtocol, + + /// The URI to send requests to. + /// + /// Must include a scheme (`http://` or `https://`) and a port. + /// + /// # Examples + /// + /// - `http://localhost:5318/v1/logs` (HTTP) + /// - `http://localhost:4317` (gRPC) + #[configurable(metadata(docs::examples = "http://localhost:5318/v1/logs"))] + #[configurable(metadata(docs::examples = "http://localhost:4317"))] + pub uri: UriSerde, + + #[configurable(derived)] + #[serde(default)] + pub compression: Compression, + + #[configurable(derived)] + #[serde(default)] + pub request: RequestConfig, + + #[configurable(derived)] + pub tls: Option, + + #[configurable(derived)] + #[serde( + default, + deserialize_with = "crate::serde::bool_or_struct", + skip_serializing_if = "crate::serde::is_default" + )] + pub acknowledgements: AcknowledgementsConfig, } impl GenerateConfig for OpenTelemetryConfig { fn generate_config() -> toml::Value { toml::from_str(indoc! {r#" - [protocol] - type = "http" + protocol = "http" uri = "http://localhost:5318/v1/logs" encoding.codec = "json" "#}) @@ -89,23 +123,61 @@ impl GenerateConfig for OpenTelemetryConfig { impl SinkConfig for OpenTelemetryConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { match &self.protocol { - Protocol::Http(config) => config.build(cx).await, - Protocol::Grpc(config) => config.build(cx).await, + OtlpProtocol::Http { + method, + auth, + encoding, + payload_prefix, + payload_suffix, + batch, + } => { + let config = HttpSinkConfig { + uri: self + .uri + .uri + .to_string() + .as_str() + .try_into() + .map_err(|e| format!("invalid URI for HTTP sink: {e}"))?, + method: *method, + auth: auth.clone(), + headers: None, + compression: self.compression, + encoding: encoding.clone(), + payload_prefix: payload_prefix.clone(), + payload_suffix: payload_suffix.clone(), + batch: *batch, + request: self.request.clone(), + tls: self.tls.clone(), + acknowledgements: self.acknowledgements, + }; + config.build(cx).await + } + OtlpProtocol::Grpc { batch } => { + let config = GrpcSinkConfig { + uri: self.uri.clone(), + compression: self.compression, + batch: *batch, + request: self.request.clone(), + tls: self.tls.clone(), + acknowledgements: self.acknowledgements, + }; + config.build(cx).await + } } } fn input(&self) -> Input { match &self.protocol { - Protocol::Http(config) => config.input(), - Protocol::Grpc(config) => config.input(), + OtlpProtocol::Http { encoding, .. } => { + Input::new(encoding.config().1.input_type()) + } + OtlpProtocol::Grpc { .. } => Input::all(), } } fn acknowledgements(&self) -> &AcknowledgementsConfig { - match &self.protocol { - Protocol::Http(config) => config.acknowledgements(), - Protocol::Grpc(config) => config.acknowledgements(), - } + &self.acknowledgements } } diff --git a/tests/e2e/opentelemetry-logs/data/vector_default.yaml b/tests/e2e/opentelemetry-logs/data/vector_default.yaml index 1f3fd9b119d09..06f515b5729ec 100644 --- a/tests/e2e/opentelemetry-logs/data/vector_default.yaml +++ b/tests/e2e/opentelemetry-logs/data/vector_default.yaml @@ -51,19 +51,18 @@ sinks: otel_sink: inputs: [ "remap_otel" ] type: opentelemetry - protocol: - type: http - uri: http://otel-collector-sink:5318/v1/logs - method: post - encoding: - codec: json - framing: - method: newline_delimited - batch: - max_events: 1 - request: - headers: - content-type: application/json + protocol: http + uri: http://otel-collector-sink:5318/v1/logs + method: post + encoding: + codec: json + framing: + method: newline_delimited + batch: + max_events: 1 + request: + headers: + content-type: application/json otel_file_sink: type: file diff --git a/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml b/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml index 99c474d4cb888..18057cd60d9d7 100644 --- a/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml +++ b/tests/e2e/opentelemetry-logs/data/vector_grpc.yaml @@ -19,9 +19,8 @@ sinks: inputs: - source0.logs type: opentelemetry - protocol: - type: grpc - uri: http://otel-collector-sink:4317 + protocol: grpc + uri: http://otel-collector-sink:4317 otel_file_sink: type: file diff --git a/tests/e2e/opentelemetry-logs/data/vector_otlp.yaml b/tests/e2e/opentelemetry-logs/data/vector_otlp.yaml index fb2267ca131ec..b182ad502c900 100644 --- a/tests/e2e/opentelemetry-logs/data/vector_otlp.yaml +++ b/tests/e2e/opentelemetry-logs/data/vector_otlp.yaml @@ -19,11 +19,10 @@ sinks: inputs: - source0.logs type: opentelemetry - protocol: - type: http - uri: http://otel-collector-sink:5318/v1/logs - encoding: - codec: otlp + protocol: http + uri: http://otel-collector-sink:5318/v1/logs + encoding: + codec: otlp otel_file_sink: type: file diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 9a7ce2c4edc85..6282bfd45e1b0 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1,1165 +1,1129 @@ package metadata -generated: components: sinks: opentelemetry: configuration: protocol: { - description: """ - The protocol used to send data to an OpenTelemetry-compatible endpoint. - - Protocol configuration - """ - required: true - type: object: options: { - acknowledgements: { - description: """ - Controls how acknowledgements are handled for this sink. +generated: components: sinks: opentelemetry: configuration: { + acknowledgements: { + description: """ + Controls how acknowledgements are handled for this sink. - See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. + See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. - [e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ - """ - required: false - type: object: options: enabled: { - description: """ - Controls whether or not end-to-end acknowledgements are enabled. + [e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ + """ + required: false + type: object: options: enabled: { + description: """ + Controls whether or not end-to-end acknowledgements are enabled. - When enabled for a sink, any source that supports end-to-end - acknowledgements that is connected to that sink waits for events - to be acknowledged by **all connected sinks** before acknowledging them at the source. + When enabled for a sink, any source that supports end-to-end + acknowledgements that is connected to that sink waits for events + to be acknowledged by **all connected sinks** before acknowledging them at the source. - Enabling or disabling acknowledgements at the sink level takes precedence over any global - [`acknowledgements`][global_acks] configuration. + Enabling or disabling acknowledgements at the sink level takes precedence over any global + [`acknowledgements`][global_acks] configuration. - [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements - """ - required: false - type: bool: {} - } + [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements + """ + required: false + type: bool: {} } - auth: { - description: """ - Configuration of the authentication strategy for HTTP requests. + } + auth: { + description: """ + Configuration of the authentication strategy for HTTP requests. + + HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an + HTTP header without any additional encryption beyond what is provided by the transport itself. + """ + relevant_when: "protocol = \"http\"" + required: false + type: object: options: { + auth: { + description: "The AWS authentication configuration." + relevant_when: "strategy = \"aws\"" + required: true + type: object: options: { + access_key_id: { + description: "The AWS access key ID." + required: true + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] + } + assume_role: { + description: """ + The ARN of an [IAM role][iam_role] to assume. - HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an - HTTP header without any additional encryption beyond what is provided by the transport itself. - """ - relevant_when: "type = \"http\"" - required: false - type: object: options: { - auth: { - description: "The AWS authentication configuration." - relevant_when: "strategy = \"aws\"" - required: true - type: object: options: { - access_key_id: { - description: "The AWS access key ID." - required: true - type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] - } - assume_role: { - description: """ - The ARN of an [IAM role][iam_role] to assume. - - [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html - """ - required: true - type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] - } - credentials_file: { - description: "Path to the credentials file." - required: true - type: string: examples: ["/my/aws/credentials"] - } - external_id: { - description: """ - The optional unique external ID in conjunction with role to assume. - - [external_id]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html - """ - required: false - type: string: examples: ["randomEXAMPLEidString"] - } - imds: { - description: "Configuration for authenticating with AWS through IMDS." - required: false - type: object: options: { - connect_timeout_seconds: { - description: "Connect timeout for IMDS." - required: false - type: uint: { - default: 1 - unit: "seconds" - } - } - max_attempts: { - description: "Number of IMDS retries for fetching tokens and metadata." - required: false - type: uint: default: 4 - } - read_timeout_seconds: { - description: "Read timeout for IMDS." - required: false - type: uint: { - default: 1 - unit: "seconds" - } + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] + } + credentials_file: { + description: "Path to the credentials file." + required: true + type: string: examples: ["/my/aws/credentials"] + } + external_id: { + description: """ + The optional unique external ID in conjunction with role to assume. + + [external_id]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html + """ + required: false + type: string: examples: ["randomEXAMPLEidString"] + } + imds: { + description: "Configuration for authenticating with AWS through IMDS." + required: false + type: object: options: { + connect_timeout_seconds: { + description: "Connect timeout for IMDS." + required: false + type: uint: { + default: 1 + unit: "seconds" } } - } - load_timeout_secs: { - description: """ - Timeout for successfully loading any credentials, in seconds. - - Relevant when the default credentials chain or `assume_role` is used. - """ - required: false - type: uint: { - examples: [30] - unit: "seconds" + max_attempts: { + description: "Number of IMDS retries for fetching tokens and metadata." + required: false + type: uint: default: 4 } - } - profile: { - description: """ - The credentials profile to use. - - Used to select AWS credentials from a provided credentials file. - """ - required: false - type: string: { - default: "default" - examples: ["develop"] + read_timeout_seconds: { + description: "Read timeout for IMDS." + required: false + type: uint: { + default: 1 + unit: "seconds" + } } } - region: { - description: """ - The [AWS region][aws_region] to send STS requests to. - - If not set, this defaults to the configured region - for the service itself. + } + load_timeout_secs: { + description: """ + Timeout for successfully loading any credentials, in seconds. - [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints - """ - required: false - type: string: examples: ["us-west-2"] - } - secret_access_key: { - description: "The AWS secret access key." - required: true - type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] + Relevant when the default credentials chain or `assume_role` is used. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" } - session_name: { - description: """ - The optional [RoleSessionName][role_session_name] is a unique session identifier for your assumed role. - - Should be unique per principal or reason. - If not set, the session name is autogenerated like assume-role-provider-1736428351340 + } + profile: { + description: """ + The credentials profile to use. - [role_session_name]: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html - """ - required: false - type: string: examples: ["vector-indexer-role"] - } - session_token: { - description: """ - The AWS session token. - See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) - """ - required: false - type: string: examples: ["AQoDYXdz...AQoDYXdz..."] + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: { + default: "default" + examples: ["develop"] } } - } - password: { - description: "The basic authentication password." - relevant_when: "strategy = \"basic\"" - required: true - type: string: examples: ["${PASSWORD}", "password"] - } - service: { - description: "The AWS service name to use for signing." - relevant_when: "strategy = \"aws\"" - required: true - type: string: {} - } - strategy: { - description: "The authentication strategy to use." - required: true - type: string: enum: { - aws: "AWS authentication." - basic: """ - Basic authentication. + region: { + description: """ + The [AWS region][aws_region] to send STS requests to. - The username and password are concatenated and encoded using [base64][base64]. + If not set, this defaults to the configured region + for the service itself. - [base64]: https://en.wikipedia.org/wiki/Base64 - """ - bearer: """ - Bearer authentication. + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-west-2"] + } + secret_access_key: { + description: "The AWS secret access key." + required: true + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] + } + session_name: { + description: """ + The optional [RoleSessionName][role_session_name] is a unique session identifier for your assumed role. - The bearer token value (OAuth2, JWT, etc.) is passed as-is. - """ - custom: "Custom Authorization Header Value, will be inserted into the headers as `Authorization: < value >`" + Should be unique per principal or reason. + If not set, the session name is autogenerated like assume-role-provider-1736428351340 + + [role_session_name]: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html + """ + required: false + type: string: examples: ["vector-indexer-role"] + } + session_token: { + description: """ + The AWS session token. + See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + """ + required: false + type: string: examples: ["AQoDYXdz...AQoDYXdz..."] } - } - token: { - description: "The bearer authentication token." - relevant_when: "strategy = \"bearer\"" - required: true - type: string: {} - } - user: { - description: "The basic authentication username." - relevant_when: "strategy = \"basic\"" - required: true - type: string: examples: ["${USERNAME}", "username"] - } - value: { - description: "Custom string value of the Authorization header" - relevant_when: "strategy = \"custom\"" - required: true - type: string: examples: ["${AUTH_HEADER_VALUE}", "CUSTOM_PREFIX ${TOKEN}"] } } - } - batch: { - description: "Event batching behavior." - required: false - type: object: options: { - max_bytes: { - description: """ - The maximum size of a batch that is processed by a sink. + password: { + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${PASSWORD}", "password"] + } + service: { + description: "The AWS service name to use for signing." + relevant_when: "strategy = \"aws\"" + required: true + type: string: {} + } + strategy: { + description: "The authentication strategy to use." + required: true + type: string: enum: { + aws: "AWS authentication." + basic: """ + Basic authentication. + + The username and password are concatenated and encoded using [base64][base64]. - This is based on the uncompressed size of the batched events, before they are - serialized or compressed. + [base64]: https://en.wikipedia.org/wiki/Base64 """ - required: false - type: uint: { - default: 10000000 - unit: "bytes" - } + bearer: """ + Bearer authentication. + + The bearer token value (OAuth2, JWT, etc.) is passed as-is. + """ + custom: "Custom Authorization Header Value, will be inserted into the headers as `Authorization: < value >`" } - max_events: { - description: "The maximum size of a batch before it is flushed." - required: false - type: uint: unit: "events" + } + token: { + description: "The bearer authentication token." + relevant_when: "strategy = \"bearer\"" + required: true + type: string: {} + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["${USERNAME}", "username"] + } + value: { + description: "Custom string value of the Authorization header" + relevant_when: "strategy = \"custom\"" + required: true + type: string: examples: ["${AUTH_HEADER_VALUE}", "CUSTOM_PREFIX ${TOKEN}"] + } + } + } + batch: { + description: "Event batching behavior." + required: false + type: object: options: { + max_bytes: { + description: """ + The maximum size of a batch that is processed by a sink. + + This is based on the uncompressed size of the batched events, before they are + serialized or compressed. + """ + required: false + type: uint: { + default: 10000000 + unit: "bytes" } - timeout_secs: { - description: "The maximum age of a batch before it is flushed." - required: false - type: float: { - default: 1.0 - unit: "seconds" - } + } + max_events: { + description: "The maximum size of a batch before it is flushed." + required: false + type: uint: unit: "events" + } + timeout_secs: { + description: "The maximum age of a batch before it is flushed." + required: false + type: float: { + default: 1.0 + unit: "seconds" } } } - compression: { - description: """ - Compression configuration. - - All compression algorithms use the default compression level unless otherwise specified. - """ - required: false - type: string: { - default: "none" - enum: { - gzip: """ - [Gzip][gzip] compression. - - [gzip]: https://www.gzip.org/ - """ - none: "No compression." - snappy: """ - [Snappy][snappy] compression. + } + compression: { + description: """ + Compression configuration. + + All compression algorithms use the default compression level unless otherwise specified. + """ + required: false + type: string: { + default: "none" + enum: { + gzip: """ + [Gzip][gzip] compression. + + [gzip]: https://www.gzip.org/ + """ + none: "No compression." + snappy: """ + [Snappy][snappy] compression. - [snappy]: https://github.com/google/snappy/blob/main/docs/README.md - """ - zlib: """ - [Zlib][zlib] compression. + [snappy]: https://github.com/google/snappy/blob/main/docs/README.md + """ + zlib: """ + [Zlib][zlib] compression. - [zlib]: https://zlib.net/ - """ - zstd: """ - [Zstandard][zstd] compression. + [zlib]: https://zlib.net/ + """ + zstd: """ + [Zstandard][zstd] compression. - [zstd]: https://facebook.github.io/zstd/ - """ - } + [zstd]: https://facebook.github.io/zstd/ + """ } } - encoding: { - description: """ - Encoding configuration. - Configures how events are encoded into raw bytes. - The selected encoding also determines which input types (logs, metrics, traces) are supported. - """ - relevant_when: "type = \"http\"" - required: true - type: object: options: { - avro: { - description: "Apache Avro-specific encoder options." - relevant_when: "codec = \"avro\"" - required: true - type: object: options: schema: { - description: "The Avro schema." - required: true - type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] - } + } + encoding: { + description: """ + Encoding configuration. + Configures how events are encoded into raw bytes. + The selected encoding also determines which input types (logs, metrics, traces) are supported. + """ + relevant_when: "protocol = \"http\"" + required: true + type: object: options: { + avro: { + description: "Apache Avro-specific encoder options." + relevant_when: "codec = \"avro\"" + required: true + type: object: options: schema: { + description: "The Avro schema." + required: true + type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] } - cef: { - description: "The CEF Serializer Options." - relevant_when: "codec = \"cef\"" - required: true - type: object: options: { - device_event_class_id: { - description: """ - Unique identifier for each event type. Identifies the type of event reported. - The value length must be less than or equal to 1023. - """ - required: true - type: string: {} - } - device_product: { - description: """ - Identifies the product of a vendor. - The part of a unique device identifier. No two products can use the same combination of device vendor and device product. - The value length must be less than or equal to 63. - """ - required: true - type: string: {} - } - device_vendor: { - description: """ - Identifies the vendor of the product. - The part of a unique device identifier. No two products can use the same combination of device vendor and device product. - The value length must be less than or equal to 63. - """ - required: true - type: string: {} - } - device_version: { - description: """ - Identifies the version of the problem. The combination of the device product, vendor, and this value make up the unique id of the device that sends messages. - The value length must be less than or equal to 31. - """ - required: true - type: string: {} - } - extensions: { - description: """ - The collection of key-value pairs. Keys are the keys of the extensions, and values are paths that point to the extension values of a log event. - The event can have any number of key-value pairs in any order. - """ - required: false - type: object: options: "*": { - description: "This is a path that points to the extension value of a log event." - required: true - type: string: {} - } - } - name: { - description: """ - This is a path that points to the human-readable description of a log event. - The value length must be less than or equal to 512. - Equals "cef.name" by default. - """ - required: true - type: string: {} - } - severity: { - description: """ - This is a path that points to the field of a log event that reflects importance of the event. - - It must point to a number from 0 to 10. - 0 = lowest_importance, 10 = highest_importance. - Set to "cef.severity" by default. - """ - required: true + } + cef: { + description: "The CEF Serializer Options." + relevant_when: "codec = \"cef\"" + required: true + type: object: options: { + device_event_class_id: { + description: """ + Unique identifier for each event type. Identifies the type of event reported. + The value length must be less than or equal to 1023. + """ + required: true + type: string: {} + } + device_product: { + description: """ + Identifies the product of a vendor. + The part of a unique device identifier. No two products can use the same combination of device vendor and device product. + The value length must be less than or equal to 63. + """ + required: true + type: string: {} + } + device_vendor: { + description: """ + Identifies the vendor of the product. + The part of a unique device identifier. No two products can use the same combination of device vendor and device product. + The value length must be less than or equal to 63. + """ + required: true + type: string: {} + } + device_version: { + description: """ + Identifies the version of the problem. The combination of the device product, vendor, and this value make up the unique id of the device that sends messages. + The value length must be less than or equal to 31. + """ + required: true + type: string: {} + } + extensions: { + description: """ + The collection of key-value pairs. Keys are the keys of the extensions, and values are paths that point to the extension values of a log event. + The event can have any number of key-value pairs in any order. + """ + required: false + type: object: options: "*": { + description: "This is a path that points to the extension value of a log event." + required: true type: string: {} } - version: { - description: """ - CEF Version. Can be either 0 or 1. - Set to "0" by default. - """ - required: true - type: string: enum: { - V0: "CEF specification version 0.1." - V1: "CEF specification version 1.x." - } + } + name: { + description: """ + This is a path that points to the human-readable description of a log event. + The value length must be less than or equal to 512. + Equals "cef.name" by default. + """ + required: true + type: string: {} + } + severity: { + description: """ + This is a path that points to the field of a log event that reflects importance of the event. + + It must point to a number from 0 to 10. + 0 = lowest_importance, 10 = highest_importance. + Set to "cef.severity" by default. + """ + required: true + type: string: {} + } + version: { + description: """ + CEF Version. Can be either 0 or 1. + Set to "0" by default. + """ + required: true + type: string: enum: { + V0: "CEF specification version 0.1." + V1: "CEF specification version 1.x." } } } - codec: { - description: "The codec to use for encoding events." - required: true - type: string: enum: { - avro: """ - Encodes an event as an [Apache Avro][apache_avro] message. + } + codec: { + description: "The codec to use for encoding events." + required: true + type: string: enum: { + avro: """ + Encodes an event as an [Apache Avro][apache_avro] message. - [apache_avro]: https://avro.apache.org/ - """ - cef: "Encodes an event as a CEF (Common Event Format) formatted message." - csv: """ - Encodes an event as a CSV message. + [apache_avro]: https://avro.apache.org/ + """ + cef: "Encodes an event as a CEF (Common Event Format) formatted message." + csv: """ + Encodes an event as a CSV message. - This codec must be configured with fields to encode. - """ - gelf: """ - Encodes an event as a [GELF][gelf] message. + This codec must be configured with fields to encode. + """ + gelf: """ + Encodes an event as a [GELF][gelf] message. - This codec is experimental for the following reason: + This codec is experimental for the following reason: - The GELF specification is more strict than the actual Graylog receiver. - Vector's encoder currently adheres more strictly to the GELF spec, with - the exception that some characters such as `@` are allowed in field names. + The GELF specification is more strict than the actual Graylog receiver. + Vector's encoder currently adheres more strictly to the GELF spec, with + the exception that some characters such as `@` are allowed in field names. - Other GELF codecs, such as Loki's, use a [Go SDK][implementation] that is maintained - by Graylog and is much more relaxed than the GELF spec. + Other GELF codecs, such as Loki's, use a [Go SDK][implementation] that is maintained + by Graylog and is much more relaxed than the GELF spec. - Going forward, Vector will use that [Go SDK][implementation] as the reference implementation, which means - the codec might continue to relax the enforcement of the specification. + Going forward, Vector will use that [Go SDK][implementation] as the reference implementation, which means + the codec might continue to relax the enforcement of the specification. - [gelf]: https://docs.graylog.org/docs/gelf - [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go - """ - json: """ - Encodes an event as [JSON][json]. + [gelf]: https://docs.graylog.org/docs/gelf + [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go + """ + json: """ + Encodes an event as [JSON][json]. - [json]: https://www.json.org/ - """ - logfmt: """ - Encodes an event as a [logfmt][logfmt] message. + [json]: https://www.json.org/ + """ + logfmt: """ + Encodes an event as a [logfmt][logfmt] message. - [logfmt]: https://brandur.org/logfmt - """ - native: """ - Encodes an event in the [native Protocol Buffers format][vector_native_protobuf]. + [logfmt]: https://brandur.org/logfmt + """ + native: """ + Encodes an event in the [native Protocol Buffers format][vector_native_protobuf]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Encodes an event in the [native JSON format][vector_native_json]. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Encodes an event in the [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - otlp: """ - Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. - This codec uses protobuf encoding, which is the recommended format for OTLP. - The output is suitable for sending to OTLP-compatible endpoints with - `content-type: application/x-protobuf`. + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. - [otlp]: https://opentelemetry.io/docs/specs/otlp/ - """ - protobuf: """ - Encodes an event as a [Protobuf][protobuf] message. + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. - [protobuf]: https://protobuf.dev/ - """ - raw_message: """ - No encoding. + [protobuf]: https://protobuf.dev/ + """ + raw_message: """ + No encoding. - This encoding uses the `message` field of a log event. + This encoding uses the `message` field of a log event. - Be careful if you are modifying your log events (for example, by using a `remap` - transform) and removing the message field while doing additional parsing on it, as this - could lead to the encoding emitting empty strings for the given event. - """ - syslog: """ - Syslog encoding - RFC 3164 and 5424 are supported - """ - text: """ - Plain text encoding. + Be careful if you are modifying your log events (for example, by using a `remap` + transform) and removing the message field while doing additional parsing on it, as this + could lead to the encoding emitting empty strings for the given event. + """ + syslog: """ + Syslog encoding + RFC 3164 and 5424 are supported + """ + text: """ + Plain text encoding. - This encoding uses the `message` field of a log event. For metrics, it uses an - encoding that resembles the Prometheus export format. + This encoding uses the `message` field of a log event. For metrics, it uses an + encoding that resembles the Prometheus export format. - Be careful if you are modifying your log events (for example, by using a `remap` - transform) and removing the message field while doing additional parsing on it, as this - could lead to the encoding emitting empty strings for the given event. - """ - } + Be careful if you are modifying your log events (for example, by using a `remap` + transform) and removing the message field while doing additional parsing on it, as this + could lead to the encoding emitting empty strings for the given event. + """ } - csv: { - description: "The CSV Serializer Options." - relevant_when: "codec = \"csv\"" - required: true - type: object: options: { - capacity: { - description: """ - Sets the capacity (in bytes) of the internal buffer used in the CSV writer. - This defaults to 8192 bytes (8KB). - """ - required: false - type: uint: default: 8192 - } - delimiter: { - description: "The field delimiter to use when writing CSV." - required: false - type: ascii_char: default: "," - } - double_quote: { - description: """ - Enables double quote escapes. - - This is enabled by default, but you can disable it. When disabled, quotes in - field data are escaped instead of doubled. - """ - required: false - type: bool: default: true - } - escape: { - description: """ - The escape character to use when writing CSV. + } + csv: { + description: "The CSV Serializer Options." + relevant_when: "codec = \"csv\"" + required: true + type: object: options: { + capacity: { + description: """ + Sets the capacity (in bytes) of the internal buffer used in the CSV writer. + This defaults to 8192 bytes (8KB). + """ + required: false + type: uint: default: 8192 + } + delimiter: { + description: "The field delimiter to use when writing CSV." + required: false + type: ascii_char: default: "," + } + double_quote: { + description: """ + Enables double quote escapes. - In some variants of CSV, quotes are escaped using a special escape character - like \\ (instead of escaping quotes by doubling them). + This is enabled by default, but you can disable it. When disabled, quotes in + field data are escaped instead of doubled. + """ + required: false + type: bool: default: true + } + escape: { + description: """ + The escape character to use when writing CSV. - To use this, `double_quotes` needs to be disabled as well; otherwise, this setting is ignored. - """ - required: false - type: ascii_char: default: "\"" - } - fields: { - description: """ - Configures the fields that are encoded, as well as the order in which they - appear in the output. - - If a field is not present in the event, the output for that field is an empty string. - - Values of type `Array`, `Object`, and `Regex` are not supported, and the - output for any of these types is an empty string. - """ - required: true - type: array: items: type: string: {} - } - quote: { - description: "The quote character to use when writing CSV." - required: false - type: ascii_char: default: "\"" - } - quote_style: { - description: "The quoting style to use when writing CSV data." - required: false - type: string: { - default: "necessary" - enum: { - always: "Always puts quotes around every field." - necessary: """ - Puts quotes around fields only when necessary. - They are necessary when fields contain a quote, delimiter, or record terminator. - Quotes are also necessary when writing an empty record - (which is indistinguishable from a record with one empty field). - """ - never: "Never writes quotes, even if it produces invalid CSV data." - non_numeric: """ - Puts quotes around all fields that are non-numeric. - This means that when writing a field that does not parse as a valid float or integer, - quotes are used even if they aren't strictly necessary. - """ - } + In some variants of CSV, quotes are escaped using a special escape character + like \\ (instead of escaping quotes by doubling them). + + To use this, `double_quotes` needs to be disabled as well; otherwise, this setting is ignored. + """ + required: false + type: ascii_char: default: "\"" + } + fields: { + description: """ + Configures the fields that are encoded, as well as the order in which they + appear in the output. + + If a field is not present in the event, the output for that field is an empty string. + + Values of type `Array`, `Object`, and `Regex` are not supported, and the + output for any of these types is an empty string. + """ + required: true + type: array: items: type: string: {} + } + quote: { + description: "The quote character to use when writing CSV." + required: false + type: ascii_char: default: "\"" + } + quote_style: { + description: "The quoting style to use when writing CSV data." + required: false + type: string: { + default: "necessary" + enum: { + always: "Always puts quotes around every field." + necessary: """ + Puts quotes around fields only when necessary. + They are necessary when fields contain a quote, delimiter, or record terminator. + Quotes are also necessary when writing an empty record + (which is indistinguishable from a record with one empty field). + """ + never: "Never writes quotes, even if it produces invalid CSV data." + non_numeric: """ + Puts quotes around all fields that are non-numeric. + This means that when writing a field that does not parse as a valid float or integer, + quotes are used even if they aren't strictly necessary. + """ } } } } - except_fields: { - description: "List of fields that are excluded from the encoded event." + } + except_fields: { + description: "List of fields that are excluded from the encoded event." + required: false + type: array: items: type: string: {} + } + gelf: { + description: "The GELF Serializer Options." + relevant_when: "codec = \"gelf\"" + required: false + type: object: options: max_chunk_size: { + description: """ + Maximum size for each GELF chunked datagram (including 12-byte header). + Chunking starts when datagrams exceed this size. + For Graylog target, keep at or below 8192 bytes; for Vector target (`gelf` decoding with `chunked_gelf` framing), up to 65,500 bytes is recommended. + """ + required: false + type: uint: default: 8192 + } + } + json: { + description: "Options for the JsonSerializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: pretty: { + description: "Whether to use pretty JSON formatting." required: false - type: array: items: type: string: {} + type: bool: default: false } - gelf: { - description: "The GELF Serializer Options." - relevant_when: "codec = \"gelf\"" - required: false - type: object: options: max_chunk_size: { - description: """ - Maximum size for each GELF chunked datagram (including 12-byte header). - Chunking starts when datagrams exceed this size. - For Graylog target, keep at or below 8192 bytes; for Vector target (`gelf` decoding with `chunked_gelf` framing), up to 65,500 bytes is recommended. - """ - required: false - type: uint: default: 8192 + } + metric_tag_values: { + description: """ + Controls how metric tag values are encoded. + + When set to `single`, only the last non-bare value of tags are displayed with the + metric. When set to `full`, all metric tags are exposed as separate assignments. + """ + relevant_when: "codec = \"json\" or codec = \"text\"" + required: false + type: string: { + default: "single" + enum: { + full: "All tags are exposed as arrays of either string or null values." + single: """ + Tag values are exposed as single strings, the same as they were before this config + option. Tags with multiple values show the last assigned value, and null values + are ignored. + """ } } - json: { - description: "Options for the JsonSerializer." - relevant_when: "codec = \"json\"" - required: false - type: object: options: pretty: { - description: "Whether to use pretty JSON formatting." - required: false + } + only_fields: { + description: "List of fields that are included in the encoded event." + required: false + type: array: items: type: string: {} + } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + use_json_names: { + description: """ + Use JSON field names (camelCase) instead of protobuf field names (snake_case). + + When enabled, the serializer looks for fields using their JSON names as defined + in the `.proto` file (for example `jobDescription` instead of `job_description`). + + This is useful when working with data that has already been converted from JSON or + when interfacing with systems that use JSON naming conventions. + """ + required: false type: bool: default: false } } - metric_tag_values: { - description: """ - Controls how metric tag values are encoded. + } + syslog: { + description: "Options for the Syslog serializer." + relevant_when: "codec = \"syslog\"" + required: false + type: object: options: { + app_name: { + description: """ + Path to a field in the event to use for the app name. - When set to `single`, only the last non-bare value of tags are displayed with the - metric. When set to `full`, all metric tags are exposed as separate assignments. - """ - relevant_when: "codec = \"json\" or codec = \"text\"" - required: false - type: string: { - default: "single" - enum: { - full: "All tags are exposed as arrays of either string or null values." - single: """ - Tag values are exposed as single strings, the same as they were before this config - option. Tags with multiple values show the last assigned value, and null values - are ignored. - """ - } + If not provided, the encoder checks for a semantic "service" field. + If that is also missing, it defaults to "vector". + """ + required: false + type: string: {} } - } - only_fields: { - description: "List of fields that are included in the encoded event." - required: false - type: array: items: type: string: {} - } - protobuf: { - description: "Options for the Protobuf serializer." - relevant_when: "codec = \"protobuf\"" - required: true - type: object: options: { - desc_file: { - description: """ - The path to the protobuf descriptor set file. - - This file is the output of `protoc -I -o ` - - You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). - """ - required: true - type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] - } - message_type: { - description: "The name of the message type to use for serializing." - required: true - type: string: examples: ["package.Message"] - } - use_json_names: { - description: """ - Use JSON field names (camelCase) instead of protobuf field names (snake_case). - - When enabled, the serializer looks for fields using their JSON names as defined - in the `.proto` file (for example `jobDescription` instead of `job_description`). - - This is useful when working with data that has already been converted from JSON or - when interfacing with systems that use JSON naming conventions. - """ - required: false - type: bool: default: false - } + facility: { + description: "Path to a field in the event to use for the facility. Defaults to \"user\"." + required: false + type: string: {} } - } - syslog: { - description: "Options for the Syslog serializer." - relevant_when: "codec = \"syslog\"" - required: false - type: object: options: { - app_name: { - description: """ - Path to a field in the event to use for the app name. - - If not provided, the encoder checks for a semantic "service" field. - If that is also missing, it defaults to "vector". - """ - required: false - type: string: {} - } - facility: { - description: "Path to a field in the event to use for the facility. Defaults to \"user\"." - required: false - type: string: {} - } - msg_id: { - description: "Path to a field in the event to use for the msg ID." - required: false - type: string: {} - } - proc_id: { - description: "Path to a field in the event to use for the proc ID." - required: false - type: string: {} - } - rfc: { - description: "RFC to use for formatting." - required: false - type: string: { - default: "rfc5424" - enum: { - rfc3164: "The legacy RFC3164 syslog format." - rfc5424: "The modern RFC5424 syslog format." - } + msg_id: { + description: "Path to a field in the event to use for the msg ID." + required: false + type: string: {} + } + proc_id: { + description: "Path to a field in the event to use for the proc ID." + required: false + type: string: {} + } + rfc: { + description: "RFC to use for formatting." + required: false + type: string: { + default: "rfc5424" + enum: { + rfc3164: "The legacy RFC3164 syslog format." + rfc5424: "The modern RFC5424 syslog format." } } - severity: { - description: "Path to a field in the event to use for the severity. Defaults to \"informational\"." - required: false - type: string: {} - } } - } - timestamp_format: { - description: "Format used for timestamp fields." - required: false - type: string: enum: { - rfc3339: "Represent the timestamp as a RFC 3339 timestamp." - unix: "Represent the timestamp as a Unix timestamp." - unix_float: "Represent the timestamp as a Unix timestamp in floating point." - unix_ms: "Represent the timestamp as a Unix timestamp in milliseconds." - unix_ns: "Represent the timestamp as a Unix timestamp in nanoseconds." - unix_us: "Represent the timestamp as a Unix timestamp in microseconds." + severity: { + description: "Path to a field in the event to use for the severity. Defaults to \"informational\"." + required: false + type: string: {} } } } + timestamp_format: { + description: "Format used for timestamp fields." + required: false + type: string: enum: { + rfc3339: "Represent the timestamp as a RFC 3339 timestamp." + unix: "Represent the timestamp as a Unix timestamp." + unix_float: "Represent the timestamp as a Unix timestamp in floating point." + unix_ms: "Represent the timestamp as a Unix timestamp in milliseconds." + unix_ns: "Represent the timestamp as a Unix timestamp in nanoseconds." + unix_us: "Represent the timestamp as a Unix timestamp in microseconds." + } + } } - framing: { - description: "Framing configuration." - relevant_when: "type = \"http\"" - required: false - type: object: options: { - character_delimited: { - description: "Options for the character delimited encoder." - relevant_when: "method = \"character_delimited\"" - required: true - type: object: options: delimiter: { - description: "The ASCII (7-bit) character that delimits byte sequences." - required: true - type: ascii_char: {} - } + } + framing: { + description: "Framing configuration." + relevant_when: "protocol = \"http\"" + required: false + type: object: options: { + character_delimited: { + description: "Options for the character delimited encoder." + relevant_when: "method = \"character_delimited\"" + required: true + type: object: options: delimiter: { + description: "The ASCII (7-bit) character that delimits byte sequences." + required: true + type: ascii_char: {} } - length_delimited: { - description: "Options for the length delimited decoder." - relevant_when: "method = \"length_delimited\"" - required: true - type: object: options: { - length_field_is_big_endian: { - description: "Length field byte order (little or big endian)" - required: false - type: bool: default: true - } - length_field_length: { - description: "Number of bytes representing the field length" - required: false - type: uint: default: 4 - } - length_field_offset: { - description: "Number of bytes in the header before the length field" - required: false - type: uint: default: 0 - } - max_frame_length: { - description: "Maximum frame length" - required: false - type: uint: default: 8388608 - } + } + length_delimited: { + description: "Options for the length delimited decoder." + relevant_when: "method = \"length_delimited\"" + required: true + type: object: options: { + length_field_is_big_endian: { + description: "Length field byte order (little or big endian)" + required: false + type: bool: default: true + } + length_field_length: { + description: "Number of bytes representing the field length" + required: false + type: uint: default: 4 + } + length_field_offset: { + description: "Number of bytes in the header before the length field" + required: false + type: uint: default: 0 + } + max_frame_length: { + description: "Maximum frame length" + required: false + type: uint: default: 8388608 } } - max_frame_length: { - description: "Maximum frame length" - relevant_when: "method = \"varint_length_delimited\"" - required: false - type: uint: default: 8388608 - } - method: { - description: "The framing method." - required: true - type: string: enum: { - bytes: "Event data is not delimited at all." - character_delimited: "Event data is delimited by a single ASCII (7-bit) character." - length_delimited: """ - Event data is prefixed with its length in bytes. + } + max_frame_length: { + description: "Maximum frame length" + relevant_when: "method = \"varint_length_delimited\"" + required: false + type: uint: default: 8388608 + } + method: { + description: "The framing method." + required: true + type: string: enum: { + bytes: "Event data is not delimited at all." + character_delimited: "Event data is delimited by a single ASCII (7-bit) character." + length_delimited: """ + Event data is prefixed with its length in bytes. - The prefix is a 32-bit unsigned integer, little endian. - """ - newline_delimited: "Event data is delimited by a newline (LF) character." - varint_length_delimited: """ - Event data is prefixed with its length in bytes as a varint. + The prefix is a 32-bit unsigned integer, little endian. + """ + newline_delimited: "Event data is delimited by a newline (LF) character." + varint_length_delimited: """ + Event data is prefixed with its length in bytes as a varint. - This is compatible with protobuf's length-delimited encoding. - """ - } + This is compatible with protobuf's length-delimited encoding. + """ } } } - headers: { - deprecated: true - deprecated_message: "This option has been deprecated, use `request.headers` instead." - description: "A list of custom headers to add to each request." - relevant_when: "type = \"http\"" - required: false - type: object: options: "*": { - description: "An HTTP request header and it's value." - required: true - type: string: {} + } + method: { + description: "The HTTP method to use. Defaults to `post`." + relevant_when: "protocol = \"http\"" + required: false + type: string: { + default: "post" + enum: { + delete: "DELETE." + get: "GET." + head: "HEAD." + options: "OPTIONS." + patch: "PATCH." + post: "POST." + put: "PUT." + trace: "TRACE." } } - method: { - description: "The HTTP method to use when making the request." - relevant_when: "type = \"http\"" - required: false - type: string: { - default: "post" - enum: { - delete: "DELETE." - get: "GET." - head: "HEAD." - options: "OPTIONS." - patch: "PATCH." - post: "POST." - put: "PUT." - trace: "TRACE." - } - } + } + payload_prefix: { + description: "A string to prefix the payload with." + relevant_when: "protocol = \"http\"" + required: false + type: string: default: "" + } + payload_suffix: { + description: "A string to suffix the payload with." + relevant_when: "protocol = \"http\"" + required: false + type: string: default: "" + } + protocol: { + description: "The transport protocol to use." + required: true + type: string: enum: { + grpc: "Send OTLP data over gRPC." + http: "Send OTLP data over HTTP." } - payload_prefix: { - description: """ - A string to prefix the payload with. + } + request: { + description: "Outbound HTTP request settings." + required: false + type: object: options: { + adaptive_concurrency: { + description: """ + Configuration of adaptive concurrency parameters. - This option is ignored if the encoding is not character delimited JSON. + These parameters typically do not require changes from the default, and incorrect values can lead to meta-stable or + unstable performance and sink behavior. Proceed with caution. + """ + required: false + type: object: options: { + decrease_ratio: { + description: """ + The fraction of the current value to set the new concurrency limit when decreasing the limit. - If specified, the `payload_suffix` must also be specified and together they must produce a valid JSON object. - """ - relevant_when: "type = \"http\"" - required: false - type: string: { - default: "" - examples: ["{\"data\":"] - } - } - payload_suffix: { - description: """ - A string to suffix the payload with. + Valid values are greater than `0` and less than `1`. Smaller values cause the algorithm to scale back rapidly + when latency increases. - This option is ignored if the encoding is not character delimited JSON. + **Note**: The new limit is rounded down after applying this ratio. + """ + required: false + type: float: default: 0.9 + } + ewma_alpha: { + description: """ + The weighting of new measurements compared to older measurements. - If specified, the `payload_prefix` must also be specified and together they must produce a valid JSON object. - """ - relevant_when: "type = \"http\"" - required: false - type: string: { - default: "" - examples: ["}"] - } - } - request: { - description: "Outbound HTTP request settings." - required: false - type: object: options: { - adaptive_concurrency: { - description: """ - Configuration of adaptive concurrency parameters. + Valid values are greater than `0` and less than `1`. - These parameters typically do not require changes from the default, and incorrect values can lead to meta-stable or - unstable performance and sink behavior. Proceed with caution. - """ - required: false - type: object: options: { - decrease_ratio: { - description: """ - The fraction of the current value to set the new concurrency limit when decreasing the limit. - - Valid values are greater than `0` and less than `1`. Smaller values cause the algorithm to scale back rapidly - when latency increases. - - **Note**: The new limit is rounded down after applying this ratio. - """ - required: false - type: float: default: 0.9 - } - ewma_alpha: { - description: """ - The weighting of new measurements compared to older measurements. - - Valid values are greater than `0` and less than `1`. - - ARC uses an exponentially weighted moving average (EWMA) of past RTT measurements as a reference to compare with - the current RTT. Smaller values cause this reference to adjust more slowly, which may be useful if a service has - unusually high response variability. - """ - required: false - type: float: default: 0.4 - } - initial_concurrency: { - description: """ - The initial concurrency limit to use. If not specified, the initial limit is 1 (no concurrency). - - Datadog recommends setting this value to your service's average limit if you're seeing that it takes a - long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the - `adaptive_concurrency_limit` metric. - """ - required: false - type: uint: default: 1 - } - max_concurrency_limit: { - description: """ - The maximum concurrency limit. - - The adaptive request concurrency limit does not go above this bound. This is put in place as a safeguard. - """ - required: false - type: uint: default: 200 - } - rtt_deviation_scale: { - description: """ - Scale of RTT deviations which are not considered anomalous. - - Valid values are greater than or equal to `0`, and reasonable values range from `1.0` to `3.0`. - - When calculating the past RTT average, a secondary “deviation” value is also computed that indicates how variable - those values are. That deviation is used when comparing the past RTT average to the current measurements, so we - can ignore increases in RTT that are within an expected range. This factor is used to scale up the deviation to - an appropriate range. Larger values cause the algorithm to ignore larger increases in the RTT. - """ - required: false - type: float: default: 2.5 - } + ARC uses an exponentially weighted moving average (EWMA) of past RTT measurements as a reference to compare with + the current RTT. Smaller values cause this reference to adjust more slowly, which may be useful if a service has + unusually high response variability. + """ + required: false + type: float: default: 0.4 } - } - concurrency: { - description: """ - Configuration for outbound request concurrency. + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit is 1 (no concurrency). - This can be set either to one of the below enum values or to a positive integer, which denotes - a fixed concurrency limit. - """ - required: false - type: { - string: { - default: "adaptive" - enum: { - adaptive: """ - Concurrency is managed by Vector's [Adaptive Request Concurrency][arc] feature. + Datadog recommends setting this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. - [arc]: https://vector.dev/docs/architecture/arc/ - """ - none: """ - A fixed concurrency of 1. + The adaptive request concurrency limit does not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } + rtt_deviation_scale: { + description: """ + Scale of RTT deviations which are not considered anomalous. - Only one request can be outstanding at any given time. - """ - } - } - uint: {} + Valid values are greater than or equal to `0`, and reasonable values range from `1.0` to `3.0`. + + When calculating the past RTT average, a secondary “deviation” value is also computed that indicates how variable + those values are. That deviation is used when comparing the past RTT average to the current measurements, so we + can ignore increases in RTT that are within an expected range. This factor is used to scale up the deviation to + an appropriate range. Larger values cause the algorithm to ignore larger increases in the RTT. + """ + required: false + type: float: default: 2.5 } } - headers: { - description: "Additional HTTP headers to add to every HTTP request." - required: false - type: object: { - examples: [{ - Accept: "text/plain" - "X-Event-Level": "{{level}}" - "X-Event-Timestamp": "{{timestamp}}" - "X-My-Custom-Header": "A-Value" - }] - options: "*": { - description: "An HTTP request header and its value. Both header names and values support templating with event data." - required: true - type: string: {} + } + concurrency: { + description: """ + Configuration for outbound request concurrency. + + This can be set either to one of the below enum values or to a positive integer, which denotes + a fixed concurrency limit. + """ + required: false + type: { + string: { + default: "adaptive" + enum: { + adaptive: """ + Concurrency is managed by Vector's [Adaptive Request Concurrency][arc] feature. + + [arc]: https://vector.dev/docs/architecture/arc/ + """ + none: """ + A fixed concurrency of 1. + + Only one request can be outstanding at any given time. + """ } } + uint: {} } - rate_limit_duration_secs: { - description: "The time window used for the `rate_limit_num` option." - required: false - type: uint: { - default: 1 - unit: "seconds" + } + headers: { + description: "Additional HTTP headers to add to every HTTP request." + required: false + type: object: { + examples: [{ + Accept: "text/plain" + "X-Event-Level": "{{level}}" + "X-Event-Timestamp": "{{timestamp}}" + "X-My-Custom-Header": "A-Value" + }] + options: "*": { + description: "An HTTP request header and its value. Both header names and values support templating with event data." + required: true + type: string: {} } } - rate_limit_num: { - description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." - required: false - type: uint: { - default: 9223372036854775807 - unit: "requests" - } + } + rate_limit_duration_secs: { + description: "The time window used for the `rate_limit_num` option." + required: false + type: uint: { + default: 1 + unit: "seconds" } - retry_attempts: { - description: "The maximum number of retries to make for failed requests." - required: false - type: uint: { - default: 9223372036854775807 - unit: "retries" - } + } + rate_limit_num: { + description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." + required: false + type: uint: { + default: 9223372036854775807 + unit: "requests" } - retry_initial_backoff_secs: { - description: """ - The amount of time to wait before attempting the first retry for a failed request. - - After the first retry has failed, the Fibonacci sequence is used to select future backoffs. - """ - required: false - type: uint: { - default: 1 - unit: "seconds" - } + } + retry_attempts: { + description: "The maximum number of retries to make for failed requests." + required: false + type: uint: { + default: 9223372036854775807 + unit: "retries" } - retry_jitter_mode: { - description: "The jitter mode to use for retry backoff behavior." - required: false - type: string: { - default: "Full" - enum: { - Full: """ - Full jitter. - - The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff - strategy. + } + retry_initial_backoff_secs: { + description: """ + The amount of time to wait before attempting the first retry for a failed request. - Incorporating full jitter into your backoff strategy can greatly reduce the likelihood - of creating accidental denial of service (DoS) conditions against your own systems when - many clients are recovering from a failure state. - """ - None: "No jitter." - } - } + After the first retry has failed, the Fibonacci sequence is used to select future backoffs. + """ + required: false + type: uint: { + default: 1 + unit: "seconds" } - retry_max_duration_secs: { - description: "The maximum amount of time to wait between retries." - required: false - type: uint: { - default: 30 - unit: "seconds" + } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." } } - timeout_secs: { - description: """ - The time a request can take before being aborted. + } + retry_max_duration_secs: { + description: "The maximum amount of time to wait between retries." + required: false + type: uint: { + default: 30 + unit: "seconds" + } + } + timeout_secs: { + description: """ + The time a request can take before being aborted. - Datadog highly recommends that you do not lower this value below the service's internal timeout, as this could - create orphaned requests, pile on retries, and result in duplicate data downstream. - """ - required: false - type: uint: { - default: 60 - unit: "seconds" - } + Datadog highly recommends that you do not lower this value below the service's internal timeout, as this could + create orphaned requests, pile on retries, and result in duplicate data downstream. + """ + required: false + type: uint: { + default: 60 + unit: "seconds" } } } - tls: { - description: "TLS configuration." - required: false - type: object: options: { - alpn_protocols: { - description: """ - Sets the list of supported ALPN protocols. + } + tls: { + description: "TLS configuration." + required: false + type: object: options: { + alpn_protocols: { + description: """ + Sets the list of supported ALPN protocols. - Declare the supported ALPN protocols, which are used during negotiation with a peer. They are prioritized in the order - that they are defined. - """ - required: false - type: array: items: type: string: examples: ["h2"] - } - ca_file: { - description: """ - Absolute path to an additional CA certificate file. + Declare the supported ALPN protocols, which are used during negotiation with a peer. They are prioritized in the order + that they are defined. + """ + required: false + type: array: items: type: string: examples: ["h2"] + } + ca_file: { + description: """ + Absolute path to an additional CA certificate file. - The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. - """ - required: false - type: string: examples: ["/path/to/certificate_authority.crt"] - } - crt_file: { - description: """ - Absolute path to a certificate file used to identify this server. + The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + crt_file: { + description: """ + Absolute path to a certificate file used to identify this server. - The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as - an inline string in PEM format. + The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as + an inline string in PEM format. - If this is set _and_ is not a PKCS#12 archive, `key_file` must also be set. - """ - required: false - type: string: examples: ["/path/to/host_certificate.crt"] - } - key_file: { - description: """ - Absolute path to a private key file used to identify this server. + If this is set _and_ is not a PKCS#12 archive, `key_file` must also be set. + """ + required: false + type: string: examples: ["/path/to/host_certificate.crt"] + } + key_file: { + description: """ + Absolute path to a private key file used to identify this server. - The key must be in DER or PEM (PKCS#8) format. Additionally, the key can be provided as an inline string in PEM format. - """ - required: false - type: string: examples: ["/path/to/host_certificate.key"] - } - key_pass: { - description: """ - Passphrase used to unlock the encrypted key file. + The key must be in DER or PEM (PKCS#8) format. Additionally, the key can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/host_certificate.key"] + } + key_pass: { + description: """ + Passphrase used to unlock the encrypted key file. - This has no effect unless `key_file` is set. - """ - required: false - type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"] - } - server_name: { - description: """ - Server name to use when using Server Name Indication (SNI). + This has no effect unless `key_file` is set. + """ + required: false + type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"] + } + server_name: { + description: """ + Server name to use when using Server Name Indication (SNI). - Only relevant for outgoing connections. - """ - required: false - type: string: examples: ["www.example.com"] - } - verify_certificate: { - description: """ - Enables certificate verification. For components that create a server, this requires that the - client connections have a valid client certificate. For components that initiate requests, - this validates that the upstream has a valid certificate. + Only relevant for outgoing connections. + """ + required: false + type: string: examples: ["www.example.com"] + } + verify_certificate: { + description: """ + Enables certificate verification. For components that create a server, this requires that the + client connections have a valid client certificate. For components that initiate requests, + this validates that the upstream has a valid certificate. - If enabled, certificates must not be expired and must be issued by a trusted - issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the - certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and - so on, until the verification process reaches a root certificate. + If enabled, certificates must not be expired and must be issued by a trusted + issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the + certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and + so on, until the verification process reaches a root certificate. - Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. - """ - required: false - type: bool: {} - } - verify_hostname: { - description: """ - Enables hostname verification. + Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + """ + required: false + type: bool: {} + } + verify_hostname: { + description: """ + Enables hostname verification. - If enabled, the hostname used to connect to the remote host must be present in the TLS certificate presented by - the remote host, either as the Common Name or as an entry in the Subject Alternative Name extension. + If enabled, the hostname used to connect to the remote host must be present in the TLS certificate presented by + the remote host, either as the Common Name or as an entry in the Subject Alternative Name extension. - Only relevant for outgoing connections. + Only relevant for outgoing connections. - Do NOT set this to `false` unless you understand the risks of not verifying the remote hostname. - """ - required: false - type: bool: {} - } - } - } - type: { - description: "The communication protocol." - required: true - type: string: enum: { - grpc: "Send data over gRPC." - http: "Send data over HTTP." + Do NOT set this to `false` unless you understand the risks of not verifying the remote hostname. + """ + required: false + type: bool: {} } } - uri: { - description: """ - The full URI to make HTTP requests to. + } + uri: { + description: """ + The URI to send requests to. - This should include the protocol and host, but can also include the port, path, and any other valid part of a URI. - """ - required: true - type: string: { - examples: ["https://10.22.212.22:9000/endpoint"] - syntax: "template" - } - } + Must include a scheme (`http://` or `https://`) and a port. + + # Examples + + - `http://localhost:5318/v1/logs` (HTTP) + - `http://localhost:4317` (gRPC) + """ + required: true + type: string: examples: ["http://localhost:5318/v1/logs", "http://localhost:4317"] } } From 9b91117aa29a8ef8a7b9a13b8dbf10e6c61287cd Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 23 Mar 2026 17:27:23 -0400 Subject: [PATCH 08/68] Fix changelog --- changelog.d/opentelemetry_sink_grpc.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/opentelemetry_sink_grpc.feature.md b/changelog.d/opentelemetry_sink_grpc.feature.md index b115672f24dfa..78cc87a94b717 100644 --- a/changelog.d/opentelemetry_sink_grpc.feature.md +++ b/changelog.d/opentelemetry_sink_grpc.feature.md @@ -1,3 +1,3 @@ -Added gRPC transport support for the `opentelemetry` sink. Configure with `protocol.type = "grpc"` and set `protocol.endpoint` to your OTLP/gRPC endpoint (e.g. `http://localhost:4317`). Supports TLS and gzip compression. +Added gRPC transport support for the `opentelemetry` sink. Configure with `protocol = "grpc"` and set `uri` to your OTLP/gRPC endpoint (e.g. `http://localhost:4317`). Supports TLS and gzip compression. authors: thomasqueirozb From 8eb5ee573eaf33be41f322cbc3e086f9c90ff20e Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 12:29:55 -0400 Subject: [PATCH 09/68] feat(opentelemetry sink): restructure config with flat protocol enum and GrpcCompression type --- src/sinks/opentelemetry/grpc.rs | 33 ++++++++++++++++++--------------- src/sinks/opentelemetry/mod.rs | 18 +++++++++++++----- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 43b492f6a98e9..07be60ff60af1 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -43,7 +43,7 @@ use crate::{ sinks::{ Healthcheck, VectorSink, util::{ - BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, + BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, StreamSink, UriSerde, http::RequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, }, @@ -51,6 +51,21 @@ use crate::{ tls::{MaybeTlsSettings, TlsConfig}, }; +/// Compression codec for gRPC transport. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum GrpcCompression { + /// No compression. + #[default] + None, + + /// [Gzip][gzip] compression. + /// + /// [gzip]: https://www.gzip.org/ + Gzip, +} + pub(super) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { if uri.scheme().is_none() { let mut parts = uri.into_parts(); @@ -95,12 +110,9 @@ pub struct GrpcSinkConfig { #[configurable(metadata(docs::examples = "http://localhost:4317"))] pub uri: UriSerde, - /// Compression codec for outgoing gRPC requests. - /// - /// Only `none` and `gzip` are supported for gRPC transport. #[configurable(derived)] #[serde(default)] - pub compression: Compression, + pub compression: GrpcCompression, #[configurable(derived)] #[serde(default)] @@ -133,16 +145,7 @@ impl GrpcSinkConfig { MaybeTlsSettings::Raw(()) }; - let use_gzip = match self.compression { - Compression::None => false, - Compression::Gzip(_) => true, - other => { - return Err(format!( - "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" - ) - .into()) - } - }; + let use_gzip = self.compression == GrpcCompression::Gzip; let client = new_grpc_client(&tls, cx.proxy())?; let service = OtlpGrpcService::new(client, uri, use_gzip); diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 8862ce635363b..8ea6880a4af8e 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -22,7 +22,7 @@ use crate::{ tls::TlsConfig, }; -pub use grpc::GrpcSinkConfig; +pub use grpc::{GrpcCompression, GrpcSinkConfig}; /// Transport protocol for the OpenTelemetry sink. #[configurable_component] @@ -154,9 +154,19 @@ impl SinkConfig for OpenTelemetryConfig { config.build(cx).await } OtlpProtocol::Grpc { batch } => { + let grpc_compression = match self.compression { + Compression::None => GrpcCompression::None, + Compression::Gzip(_) => GrpcCompression::Gzip, + other => { + return Err(format!( + "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" + ) + .into()) + } + }; let config = GrpcSinkConfig { uri: self.uri.clone(), - compression: self.compression, + compression: grpc_compression, batch: *batch, request: self.request.clone(), tls: self.tls.clone(), @@ -169,9 +179,7 @@ impl SinkConfig for OpenTelemetryConfig { fn input(&self) -> Input { match &self.protocol { - OtlpProtocol::Http { encoding, .. } => { - Input::new(encoding.config().1.input_type()) - } + OtlpProtocol::Http { encoding, .. } => Input::new(encoding.config().1.input_type()), OtlpProtocol::Grpc { .. } => Input::all(), } } From b9c6cb740bc63db22067a1465f640eb5c84ec01c Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 12:54:09 -0400 Subject: [PATCH 10/68] fix(opentelemetry sink): split mixed-signal batches into per-signal requests to prevent retry duplicates --- src/sinks/opentelemetry/grpc.rs | 210 +++++++++++++++++--------------- 1 file changed, 115 insertions(+), 95 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 07be60ff60af1..b562294b01e73 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -225,16 +225,24 @@ pub enum OtlpGrpcError { // ── Request/Response ───────────────────────────────────────────────────────── -/// A grouped batch of OTLP events, with one proto request per signal type. -#[derive(Clone, Default)] +/// A single-signal OTLP export request. One request per signal type ensures +/// that retries are atomic — a failed metrics export cannot duplicate a +/// previously-accepted logs export. +#[derive(Clone)] pub struct OtlpGrpcRequest { - pub logs: Option, - pub metrics: Option, - pub traces: Option, + pub signal: OtlpSignalRequest, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, } +/// The OTLP export payload for a single signal type. +#[derive(Clone)] +pub enum OtlpSignalRequest { + Logs(ExportLogsServiceRequest), + Metrics(ExportMetricsServiceRequest), + Traces(ExportTraceServiceRequest), +} + impl Finalizable for OtlpGrpcRequest { fn take_finalizers(&mut self) -> EventFinalizers { self.finalizers.take_finalizers() @@ -325,34 +333,35 @@ impl Service for OtlpGrpcService { let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); let future = async move { - let mut total_bytes: usize = 0; - - if let Some(logs_req) = req.logs { - total_bytes += logs_req.encoded_len(); - svc.logs_client - .export(logs_req) - .map_err(|source| OtlpGrpcError::Request { source }) - .await?; - } - - if let Some(metrics_req) = req.metrics { - total_bytes += metrics_req.encoded_len(); - svc.metrics_client - .export(metrics_req) - .map_err(|source| OtlpGrpcError::Request { source }) - .await?; - } - - if let Some(traces_req) = req.traces { - total_bytes += traces_req.encoded_len(); - svc.traces_client - .export(traces_req) - .map_err(|source| OtlpGrpcError::Request { source }) - .await?; - } + let byte_size = match req.signal { + OtlpSignalRequest::Logs(r) => { + let len = r.encoded_len(); + svc.logs_client + .export(r) + .map_err(|source| OtlpGrpcError::Request { source }) + .await?; + len + } + OtlpSignalRequest::Metrics(r) => { + let len = r.encoded_len(); + svc.metrics_client + .export(r) + .map_err(|source| OtlpGrpcError::Request { source }) + .await?; + len + } + OtlpSignalRequest::Traces(r) => { + let len = r.encoded_len(); + svc.traces_client + .export(r) + .map_err(|source| OtlpGrpcError::Request { source }) + .await?; + len + } + }; emit!(EndpointBytesSent { - byte_size: total_bytes, + byte_size, protocol: &svc.protocol, endpoint: &svc.endpoint, }); @@ -422,16 +431,23 @@ impl OtlpSignal { } } -/// Accumulator for a batch of OTLP events, merged per signal type. -#[derive(Default)] -struct OtlpBatch { +/// Per-signal accumulator tracking the merged proto request and its associated +/// event metadata. Kept separate so each signal can be retried independently. +struct SignalData { + request: R, finalizers: EventFinalizers, event_count: usize, - events_byte_size: usize, - events_json_byte_size: GroupedCountByteSize, - logs: Option, - metrics: Option, - traces: Option, + byte_size: usize, + json_byte_size: GroupedCountByteSize, +} + +/// Accumulator for a batch of OTLP events, separated by signal type so that +/// the resulting requests can be retried independently. +#[derive(Default)] +struct OtlpBatch { + logs: Option>, + metrics: Option>, + traces: Option>, } impl Clone for OtlpBatch { @@ -490,65 +506,69 @@ where .batched(self.batch_settings.as_reducer_config( |data: &OtlpEventData| data.signal.encoded_len(), BatchReduce::new(|batch: &mut OtlpBatch, item: OtlpEventData| { - batch.finalizers.merge(item.finalizers); - batch.event_count += 1; - batch.events_byte_size += item.byte_size; - batch.events_json_byte_size += item.json_byte_size; - match item.signal { - OtlpSignal::Logs(req) => { - if let Some(existing) = &mut batch.logs { - existing.resource_logs.extend(req.resource_logs); - } else { - batch.logs = Some(req); - } - } - OtlpSignal::Metrics(req) => { - if let Some(existing) = &mut batch.metrics { - existing.resource_metrics.extend(req.resource_metrics); - } else { - batch.metrics = Some(req); + macro_rules! accumulate { + ($field:ident, $req:expr, $merge:expr) => { + match &mut batch.$field { + Some(existing) => { + $merge(&mut existing.request, $req); + existing.finalizers.merge(item.finalizers); + existing.event_count += 1; + existing.byte_size += item.byte_size; + existing.json_byte_size += item.json_byte_size; + } + slot => { + *slot = Some(SignalData { + request: $req, + finalizers: item.finalizers, + event_count: 1, + byte_size: item.byte_size, + json_byte_size: item.json_byte_size, + }); + } } - } - OtlpSignal::Traces(req) => { - if let Some(existing) = &mut batch.traces { - existing.resource_spans.extend(req.resource_spans); - } else { - batch.traces = Some(req); - } - } + }; + } + match item.signal { + OtlpSignal::Logs(req) => accumulate!(logs, req, |e: &mut ExportLogsServiceRequest, r: ExportLogsServiceRequest| { + e.resource_logs.extend(r.resource_logs) + }), + OtlpSignal::Metrics(req) => accumulate!(metrics, req, |e: &mut ExportMetricsServiceRequest, r: ExportMetricsServiceRequest| { + e.resource_metrics.extend(r.resource_metrics) + }), + OtlpSignal::Traces(req) => accumulate!(traces, req, |e: &mut ExportTraceServiceRequest, r: ExportTraceServiceRequest| { + e.resource_spans.extend(r.resource_spans) + }), } }), )) - .map(|batch| { - let builder = RequestMetadataBuilder::new( - batch.event_count, - batch.events_byte_size, - batch.events_json_byte_size, - ); - - let byte_size = batch - .logs - .as_ref() - .map_or(0, |r| r.encoded_len()) - + batch - .metrics - .as_ref() - .map_or(0, |r| r.encoded_len()) - + batch - .traces - .as_ref() - .map_or(0, |r| r.encoded_len()); - - let bytes_len = - NonZeroUsize::new(byte_size.max(1)).expect("should be non-zero"); - - OtlpGrpcRequest { - logs: batch.logs, - metrics: batch.metrics, - traces: batch.traces, - finalizers: batch.finalizers, - metadata: builder.with_request_size(bytes_len), + .flat_map(|batch| { + let mut requests = Vec::new(); + + macro_rules! push_signal { + ($field:ident, $variant:ident) => { + if let Some(data) = batch.$field { + let byte_size = data.request.encoded_len(); + let bytes_len = + NonZeroUsize::new(byte_size.max(1)).expect("should be non-zero"); + let builder = RequestMetadataBuilder::new( + data.event_count, + data.byte_size, + data.json_byte_size, + ); + requests.push(OtlpGrpcRequest { + signal: OtlpSignalRequest::$variant(data.request), + finalizers: data.finalizers, + metadata: builder.with_request_size(bytes_len), + }); + } + }; } + + push_signal!(logs, Logs); + push_signal!(metrics, Metrics); + push_signal!(traces, Traces); + + futures::stream::iter(requests) }) .into_driver(self.service) .run() @@ -648,7 +668,7 @@ mod tests { ) .unwrap(); assert_eq!(config.uri.uri.to_string(), "http://localhost:4317"); - assert_eq!(config.compression, Compression::default()); + assert_eq!(config.compression, GrpcCompression::default()); } #[test] @@ -664,7 +684,7 @@ mod tests { config.uri.uri.to_string(), "https://otelcol.example.com:4317" ); - assert!(matches!(config.compression, Compression::Gzip(_))); + assert_eq!(config.compression, GrpcCompression::Gzip); } #[test] From b3b175d7adab303b9fb0b15f74cdd5e092abf19d Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 14:00:52 -0400 Subject: [PATCH 11/68] fix(opentelemetry sink): implement real gRPC healthcheck using standard health protocol --- Cargo.toml | 2 +- src/sinks/opentelemetry/grpc.rs | 45 ++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ff78f94ac4a01..74152913c2b22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -909,7 +909,7 @@ sinks-mqtt = ["dep:rumqttc"] sinks-nats = ["dep:async-nats", "dep:nkeys"] sinks-new_relic_logs = ["sinks-http"] sinks-new_relic = [] -sinks-opentelemetry = ["sinks-http", "codecs-opentelemetry", "dep:tonic", "dep:prost"] +sinks-opentelemetry = ["sinks-http", "codecs-opentelemetry", "dep:tonic", "dep:tonic-health", "dep:prost"] sinks-papertrail = ["dep:syslog"] sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"] sinks-postgres = ["dep:sqlx"] diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index b562294b01e73..f13d01a9f31bc 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -36,7 +36,7 @@ use vector_lib::{ }; use crate::{ - config::{AcknowledgementsConfig, Input, SinkContext}, + config::{AcknowledgementsConfig, Input, SinkContext, SinkHealthcheckOptions}, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, internal_events::EndpointBytesSent, @@ -147,9 +147,9 @@ impl GrpcSinkConfig { let use_gzip = self.compression == GrpcCompression::Gzip; let client = new_grpc_client(&tls, cx.proxy())?; - let service = OtlpGrpcService::new(client, uri, use_gzip); + let service = OtlpGrpcService::new(client.clone(), uri.clone(), use_gzip); - let healthcheck = Box::pin(async move { Ok(()) }); + let healthcheck = Box::pin(grpc_healthcheck(client, uri, cx.healthcheck)); let request_settings = self.request.tower.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; @@ -183,6 +183,45 @@ fn new_grpc_client( Ok(hyper::Client::builder().http2_only(true).build(proxy)) } +async fn grpc_healthcheck( + client: hyper::Client>, BoxBody>, + uri: Uri, + options: SinkHealthcheckOptions, +) -> crate::Result<()> { + if !options.enabled { + return Ok(()); + } + + use tonic::Code; + use tonic_health::pb::{HealthCheckRequest, health_client::HealthClient}; + + let svc = HyperSvc { + uri, + client, + }; + let mut health_client = HealthClient::new(svc); + + match health_client + .check(HealthCheckRequest { + service: String::new(), + }) + .await + { + Ok(response) => { + use tonic_health::pb::health_check_response::ServingStatus; + let status = response.into_inner().status; + if status == ServingStatus::Serving as i32 { + Ok(()) + } else { + Err(format!("gRPC collector reported non-serving status: {status}").into()) + } + } + // Server is reachable but does not implement the health protocol — treat as healthy. + Err(status) if status.code() == Code::Unimplemented => Ok(()), + Err(status) => Err(Box::new(OtlpGrpcError::Request { source: status })), + } +} + // ── Retry logic ────────────────────────────────────────────────────────────── #[derive(Debug, Clone)] From c5883cd6d83777387dfb2b63a19875333dcbe458 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 14:30:49 -0400 Subject: [PATCH 12/68] fix(opentelemetry sink): make uri a Template to restore template syntax support --- src/sinks/opentelemetry/mod.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 8ea6880a4af8e..71c20084d6d82 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -16,9 +16,10 @@ use crate::{ http::config::{HttpMethod, HttpSinkConfig}, util::{ BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, - RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::RequestConfig, + RealtimeSizeBasedDefaultBatchSettings, http::RequestConfig, }, }, + template::Template, tls::TlsConfig, }; @@ -77,6 +78,7 @@ pub struct OpenTelemetryConfig { /// The URI to send requests to. /// + /// Supports template syntax (e.g. `http://{{ host }}:4318/v1/logs`). /// Must include a scheme (`http://` or `https://`) and a port. /// /// # Examples @@ -85,7 +87,7 @@ pub struct OpenTelemetryConfig { /// - `http://localhost:4317` (gRPC) #[configurable(metadata(docs::examples = "http://localhost:5318/v1/logs"))] #[configurable(metadata(docs::examples = "http://localhost:4317"))] - pub uri: UriSerde, + pub uri: Template, #[configurable(derived)] #[serde(default)] @@ -132,13 +134,7 @@ impl SinkConfig for OpenTelemetryConfig { batch, } => { let config = HttpSinkConfig { - uri: self - .uri - .uri - .to_string() - .as_str() - .try_into() - .map_err(|e| format!("invalid URI for HTTP sink: {e}"))?, + uri: self.uri.clone(), method: *method, auth: auth.clone(), headers: None, @@ -164,8 +160,13 @@ impl SinkConfig for OpenTelemetryConfig { .into()) } }; + let uri = self + .uri + .get_ref() + .parse() + .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?; let config = GrpcSinkConfig { - uri: self.uri.clone(), + uri, compression: grpc_compression, batch: *batch, request: self.request.clone(), From efc5fc2b89c19e07627ec6af1dbdf41c32d0acbb Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 14:42:45 -0400 Subject: [PATCH 13/68] feat(opentelemetry sink): use Template for gRPC uri field --- src/sinks/opentelemetry/grpc.rs | 20 ++++++++++++-------- src/sinks/opentelemetry/mod.rs | 7 +------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index f13d01a9f31bc..5cb2df661250a 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -44,10 +44,11 @@ use crate::{ Healthcheck, VectorSink, util::{ BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, - SinkBuilderExt, StreamSink, UriSerde, http::RequestConfig, + SinkBuilderExt, StreamSink, http::RequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, }, }, + template::Template, tls::{MaybeTlsSettings, TlsConfig}, }; @@ -108,7 +109,7 @@ pub struct GrpcSinkConfig { /// - `http://localhost:4317` /// - `https://otelcol.example.com:4317` #[configurable(metadata(docs::examples = "http://localhost:4317"))] - pub uri: UriSerde, + pub uri: Template, #[configurable(derived)] #[serde(default)] @@ -138,7 +139,13 @@ pub struct GrpcSinkConfig { impl GrpcSinkConfig { pub async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { // Determine TLS from the URI scheme; fall back to https when tls options are present. - let uri = with_default_scheme(self.uri.uri.clone(), self.tls.is_some())?; + let uri = with_default_scheme( + self.uri + .get_ref() + .parse() + .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?, + self.tls.is_some(), + )?; let tls = if uri.scheme_str() == Some("https") { MaybeTlsSettings::tls_client(self.tls.as_ref())? } else { @@ -706,7 +713,7 @@ mod tests { "#, ) .unwrap(); - assert_eq!(config.uri.uri.to_string(), "http://localhost:4317"); + assert_eq!(config.uri.get_ref(), "http://localhost:4317"); assert_eq!(config.compression, GrpcCompression::default()); } @@ -719,10 +726,7 @@ mod tests { "#, ) .unwrap(); - assert_eq!( - config.uri.uri.to_string(), - "https://otelcol.example.com:4317" - ); + assert_eq!(config.uri.get_ref(), "https://otelcol.example.com:4317"); assert_eq!(config.compression, GrpcCompression::Gzip); } diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 71c20084d6d82..4d2d09106ad3c 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -160,13 +160,8 @@ impl SinkConfig for OpenTelemetryConfig { .into()) } }; - let uri = self - .uri - .get_ref() - .parse() - .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?; let config = GrpcSinkConfig { - uri, + uri: self.uri.clone(), compression: grpc_compression, batch: *batch, request: self.request.clone(), From 787df5b3ffab1abf06df1210989a92b35c6a25cd Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 14:43:35 -0400 Subject: [PATCH 14/68] Format --- src/sinks/opentelemetry/grpc.rs | 10 +++------- src/sinks/opentelemetry/mod.rs | 10 ++++------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 5cb2df661250a..b203f24aa0729 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -43,9 +43,8 @@ use crate::{ sinks::{ Healthcheck, VectorSink, util::{ - BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, - SinkBuilderExt, StreamSink, http::RequestConfig, - metadata::RequestMetadataBuilder, retries::RetryLogic, + BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, + StreamSink, http::RequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, }, }, template::Template, @@ -202,10 +201,7 @@ async fn grpc_healthcheck( use tonic::Code; use tonic_health::pb::{HealthCheckRequest, health_client::HealthClient}; - let svc = HyperSvc { - uri, - client, - }; + let svc = HyperSvc { uri, client }; let mut health_client = HealthClient::new(svc); match health_client diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 4d2d09106ad3c..52564a899d4b6 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -153,12 +153,10 @@ impl SinkConfig for OpenTelemetryConfig { let grpc_compression = match self.compression { Compression::None => GrpcCompression::None, Compression::Gzip(_) => GrpcCompression::Gzip, - other => { - return Err(format!( - "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" - ) - .into()) - } + other => return Err(format!( + "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" + ) + .into()), }; let config = GrpcSinkConfig { uri: self.uri.clone(), From 2c0093bf724ec303ab198c160bdf0ed7e30dcbb6 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 14:45:31 -0400 Subject: [PATCH 15/68] Regenerate docs --- .../reference/components/sinks/generated/opentelemetry.cue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 6282bfd45e1b0..15b654b21f86b 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1116,6 +1116,7 @@ generated: components: sinks: opentelemetry: configuration: { description: """ The URI to send requests to. + Supports template syntax (e.g. `http://{{ host }}:4318/v1/logs`). Must include a scheme (`http://` or `https://`) and a port. # Examples @@ -1124,6 +1125,9 @@ generated: components: sinks: opentelemetry: configuration: { - `http://localhost:4317` (gRPC) """ required: true - type: string: examples: ["http://localhost:5318/v1/logs", "http://localhost:4317"] + type: string: { + examples: ["http://localhost:5318/v1/logs", "http://localhost:4317"] + syntax: "template" + } } } From 97a9070d00adcd9c259692e85269b603085e03aa Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 15:38:10 -0400 Subject: [PATCH 16/68] Fix markdown --- changelog.d/opentelemetry_sink_config.breaking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/opentelemetry_sink_config.breaking.md b/changelog.d/opentelemetry_sink_config.breaking.md index 33104d98860a7..b1cf6003d9027 100644 --- a/changelog.d/opentelemetry_sink_config.breaking.md +++ b/changelog.d/opentelemetry_sink_config.breaking.md @@ -4,7 +4,7 @@ configuration. Before: -``` +```yaml sinks: otel_sink: inputs: @@ -26,7 +26,7 @@ sinks: After: -``` +```yaml sinks: otel_sink: inputs: From e24bee9d7b8ba661791fec4115add23379a12f3d Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 16:53:12 -0400 Subject: [PATCH 17/68] fix(opentelemetry sink): forward request headers as gRPC metadata, restrict input type, reject template URIs for gRPC --- src/sinks/opentelemetry/grpc.rs | 78 ++++++++++++------- src/sinks/opentelemetry/mod.rs | 4 +- .../sinks/generated/opentelemetry.cue | 4 +- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index b203f24aa0729..85b5cb5a5e15d 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -36,7 +36,7 @@ use vector_lib::{ }; use crate::{ - config::{AcknowledgementsConfig, Input, SinkContext, SinkHealthcheckOptions}, + config::{AcknowledgementsConfig, DataType, Input, SinkContext, SinkHealthcheckOptions}, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, internal_events::EndpointBytesSent, @@ -137,7 +137,14 @@ pub struct GrpcSinkConfig { impl GrpcSinkConfig { pub async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { - // Determine TLS from the URI scheme; fall back to https when tls options are present. + if self.uri.is_dynamic() { + return Err( + "template syntax is not supported for `uri` with the gRPC transport; \ + use a literal URI (e.g. `http://localhost:4317`)" + .into(), + ); + } + let uri = with_default_scheme( self.uri .get_ref() @@ -145,6 +152,7 @@ impl GrpcSinkConfig { .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?, self.tls.is_some(), )?; + let tls = if uri.scheme_str() == Some("https") { MaybeTlsSettings::tls_client(self.tls.as_ref())? } else { @@ -152,8 +160,27 @@ impl GrpcSinkConfig { }; let use_gzip = self.compression == GrpcCompression::Gzip; + + let grpc_headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )> = self + .request + .headers + .iter() + .filter_map(|(k, v)| { + let key = tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) + .map_err(|e| warn!("Skipping invalid gRPC metadata key {k:?}: {e}")) + .ok()?; + let value = tonic::metadata::AsciiMetadataValue::try_from(v.as_str()) + .map_err(|e| warn!("Skipping invalid gRPC metadata value for {k:?}: {e}")) + .ok()?; + Some((key, value)) + }) + .collect(); + let client = new_grpc_client(&tls, cx.proxy())?; - let service = OtlpGrpcService::new(client.clone(), uri.clone(), use_gzip); + let service = OtlpGrpcService::new(client.clone(), uri.clone(), use_gzip, grpc_headers); let healthcheck = Box::pin(grpc_healthcheck(client, uri, cx.healthcheck)); @@ -173,7 +200,9 @@ impl GrpcSinkConfig { } pub fn input(&self) -> Input { - Input::all() + // Native Vector Metric events are not supported; OTLP-encoded metrics arrive as Log + // events with a `resourceMetrics` field and are handled correctly. + Input::new(DataType::Log | DataType::Trace) } pub const fn acknowledgements(&self) -> &AcknowledgementsConfig { @@ -322,6 +351,7 @@ pub struct OtlpGrpcService { logs_client: LogsServiceClient, metrics_client: MetricsServiceClient, traces_client: TraceServiceClient, + headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, protocol: String, endpoint: String, } @@ -331,6 +361,7 @@ impl OtlpGrpcService { hyper_client: hyper::Client>, BoxBody>, uri: Uri, compression: bool, + headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, ) -> Self { let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(uri.clone()); @@ -354,6 +385,7 @@ impl OtlpGrpcService { logs_client, metrics_client, traces_client, + headers, protocol, endpoint, } @@ -375,31 +407,25 @@ impl Service for OtlpGrpcService { let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); let future = async move { - let byte_size = match req.signal { - OtlpSignalRequest::Logs(r) => { - let len = r.encoded_len(); - svc.logs_client - .export(r) - .map_err(|source| OtlpGrpcError::Request { source }) - .await?; - len - } - OtlpSignalRequest::Metrics(r) => { - let len = r.encoded_len(); - svc.metrics_client - .export(r) - .map_err(|source| OtlpGrpcError::Request { source }) - .await?; - len - } - OtlpSignalRequest::Traces(r) => { - let len = r.encoded_len(); - svc.traces_client - .export(r) + macro_rules! export { + ($client:expr, $payload:expr) => {{ + let len = $payload.encoded_len(); + let mut grpc_req = tonic::Request::new($payload); + for (key, value) in &svc.headers { + grpc_req.metadata_mut().insert(key.clone(), value.clone()); + } + $client + .export(grpc_req) .map_err(|source| OtlpGrpcError::Request { source }) .await?; len - } + }}; + } + + let byte_size = match req.signal { + OtlpSignalRequest::Logs(r) => export!(svc.logs_client, r), + OtlpSignalRequest::Metrics(r) => export!(svc.metrics_client, r), + OtlpSignalRequest::Traces(r) => export!(svc.traces_client, r), }; emit!(EndpointBytesSent { diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 29800b5bd28ad..18800c31abe28 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -78,9 +78,11 @@ pub struct OpenTelemetryConfig { /// The URI to send requests to. /// - /// Supports template syntax (e.g. `http://{{ host }}:4318/v1/logs`). /// Must include a scheme (`http://` or `https://`) and a port. /// + /// For the HTTP transport, template syntax is supported (e.g. `http://{{ host }}:4318/v1/logs`). + /// The gRPC transport requires a literal URI; template syntax is not supported. + /// /// # Examples /// /// - `http://localhost:5318/v1/logs` (HTTP) diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 15b654b21f86b..bb02c12f802da 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1116,9 +1116,11 @@ generated: components: sinks: opentelemetry: configuration: { description: """ The URI to send requests to. - Supports template syntax (e.g. `http://{{ host }}:4318/v1/logs`). Must include a scheme (`http://` or `https://`) and a port. + For the HTTP transport, template syntax is supported (e.g. `http://{{ host }}:4318/v1/logs`). + The gRPC transport requires a literal URI; template syntax is not supported. + # Examples - `http://localhost:5318/v1/logs` (HTTP) From 88727d543e42c20be923ad0e31cbfb2199843503 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 09:48:45 -0400 Subject: [PATCH 18/68] fix(opentelemetry sink): include request headers in gRPC healthcheck --- src/sinks/opentelemetry/grpc.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 85b5cb5a5e15d..f096417549027 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -180,9 +180,8 @@ impl GrpcSinkConfig { .collect(); let client = new_grpc_client(&tls, cx.proxy())?; - let service = OtlpGrpcService::new(client.clone(), uri.clone(), use_gzip, grpc_headers); - - let healthcheck = Box::pin(grpc_healthcheck(client, uri, cx.healthcheck)); + let healthcheck = Box::pin(grpc_healthcheck(client.clone(), uri.clone(), grpc_headers.clone(), cx.healthcheck)); + let service = OtlpGrpcService::new(client, uri, use_gzip, grpc_headers); let request_settings = self.request.tower.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; @@ -221,6 +220,7 @@ fn new_grpc_client( async fn grpc_healthcheck( client: hyper::Client>, BoxBody>, uri: Uri, + headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, options: SinkHealthcheckOptions, ) -> crate::Result<()> { if !options.enabled { @@ -233,11 +233,14 @@ async fn grpc_healthcheck( let svc = HyperSvc { uri, client }; let mut health_client = HealthClient::new(svc); - match health_client - .check(HealthCheckRequest { - service: String::new(), - }) - .await + let mut req = tonic::Request::new(HealthCheckRequest { + service: String::new(), + }); + for (key, value) in headers { + req.metadata_mut().insert(key, value); + } + + match health_client.check(req).await { Ok(response) => { use tonic_health::pb::health_check_response::ServingStatus; From 571583c93675bcbff86f054e4b26d2244abdc37e Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 09:57:30 -0400 Subject: [PATCH 19/68] fix(opentelemetry sink): render templated gRPC metadata values per batch --- src/sinks/opentelemetry/grpc.rs | 77 ++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index f096417549027..6035425835170 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -161,17 +161,26 @@ impl GrpcSinkConfig { let use_gzip = self.compression == GrpcCompression::Gzip; - let grpc_headers: Vec<( + // Split headers into static (literal values) and dynamic (template values). + // Static headers are pre-parsed once and used for the healthcheck and every export. + // Dynamic headers are rendered per-event so that templated fields (e.g. tenant IDs) + // resolve correctly at export time. + let (static_header_strings, dynamic_header_templates_raw) = + self.request.split_headers(); + + let parse_key = |k: &str| { + tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) + .map_err(|e| warn!("Skipping invalid gRPC metadata key {k:?}: {e}")) + .ok() + }; + + let static_grpc_headers: Vec<( tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue, - )> = self - .request - .headers + )> = static_header_strings .iter() .filter_map(|(k, v)| { - let key = tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) - .map_err(|e| warn!("Skipping invalid gRPC metadata key {k:?}: {e}")) - .ok()?; + let key = parse_key(k)?; let value = tonic::metadata::AsciiMetadataValue::try_from(v.as_str()) .map_err(|e| warn!("Skipping invalid gRPC metadata value for {k:?}: {e}")) .ok()?; @@ -179,9 +188,20 @@ impl GrpcSinkConfig { }) .collect(); + let dynamic_grpc_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)> = + dynamic_header_templates_raw + .into_iter() + .filter_map(|(k, t)| Some((parse_key(&k)?, t))) + .collect(); + let client = new_grpc_client(&tls, cx.proxy())?; - let healthcheck = Box::pin(grpc_healthcheck(client.clone(), uri.clone(), grpc_headers.clone(), cx.healthcheck)); - let service = OtlpGrpcService::new(client, uri, use_gzip, grpc_headers); + let healthcheck = Box::pin(grpc_healthcheck( + client.clone(), + uri.clone(), + static_grpc_headers.clone(), + cx.healthcheck, + )); + let service = OtlpGrpcService::new(client, uri, use_gzip, static_grpc_headers); let request_settings = self.request.tower.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; @@ -193,6 +213,7 @@ impl GrpcSinkConfig { let sink = OtlpGrpcSink { batch_settings, service, + dynamic_header_templates: dynamic_grpc_header_templates, }; Ok((VectorSink::from_event_streamsink(sink), healthcheck)) @@ -305,6 +326,8 @@ pub enum OtlpGrpcError { #[derive(Clone)] pub struct OtlpGrpcRequest { pub signal: OtlpSignalRequest, + /// Per-event rendered values for dynamic (templated) metadata headers. + pub dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, } @@ -417,6 +440,9 @@ impl Service for OtlpGrpcService { for (key, value) in &svc.headers { grpc_req.metadata_mut().insert(key.clone(), value.clone()); } + for (key, value) in &req.dynamic_headers { + grpc_req.metadata_mut().insert(key.clone(), value.clone()); + } $client .export(grpc_req) .map_err(|source| OtlpGrpcError::Request { source }) @@ -483,6 +509,7 @@ struct OtlpEventData { json_byte_size: GroupedCountByteSize, finalizers: EventFinalizers, signal: OtlpSignal, + dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, } /// Pre-decoded OTLP signal for a single event. @@ -519,6 +546,8 @@ struct OtlpBatch { logs: Option>, metrics: Option>, traces: Option>, + /// Dynamic header values captured from the first event in the batch. + dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, } impl Clone for OtlpBatch { @@ -532,6 +561,7 @@ impl Clone for OtlpBatch { pub struct OtlpGrpcSink { pub batch_settings: BatcherSettings, pub service: S, + dynamic_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)>, } impl OtlpGrpcSink @@ -546,8 +576,28 @@ where error!("Failed to create OTLP serializer: {}", e); })?; + let dynamic_header_templates = self.dynamic_header_templates.clone(); + input - .filter_map(|mut event| { + .filter_map(move |mut event| { + let dynamic_headers: Vec<_> = dynamic_header_templates + .iter() + .filter_map(|(key, template)| { + let rendered = template + .render_string(&event) + .map_err(|e| { + warn!("Failed to render gRPC metadata template for key {:?}: {e}", key.as_str()); + }) + .ok()?; + let value = tonic::metadata::AsciiMetadataValue::try_from(rendered.as_str()) + .map_err(|e| { + warn!("Rendered gRPC metadata value for key {:?} is not valid ASCII: {e}", key.as_str()); + }) + .ok()?; + Some((key.clone(), value)) + }) + .collect(); + let signal = encode_event(&mut serializer, &event); match signal { Ok(Some(signal)) => { @@ -558,6 +608,7 @@ where json_byte_size, finalizers: event.take_finalizers(), signal, + dynamic_headers, }; futures::future::ready(Some(data)) } @@ -577,6 +628,10 @@ where .batched(self.batch_settings.as_reducer_config( |data: &OtlpEventData| data.signal.encoded_len(), BatchReduce::new(|batch: &mut OtlpBatch, item: OtlpEventData| { + if batch.dynamic_headers.is_empty() { + batch.dynamic_headers = item.dynamic_headers; + } + macro_rules! accumulate { ($field:ident, $req:expr, $merge:expr) => { match &mut batch.$field { @@ -614,6 +669,7 @@ where )) .flat_map(|batch| { let mut requests = Vec::new(); + let dynamic_headers = batch.dynamic_headers; macro_rules! push_signal { ($field:ident, $variant:ident) => { @@ -628,6 +684,7 @@ where ); requests.push(OtlpGrpcRequest { signal: OtlpSignalRequest::$variant(data.request), + dynamic_headers: dynamic_headers.clone(), finalizers: data.finalizers, metadata: builder.with_request_size(bytes_len), }); From b98209cb38d0b40bbb50ab206226b2dff95704ee Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 10:13:35 -0400 Subject: [PATCH 20/68] feat(opentelemetry sink): support template syntax for gRPC URI --- src/sinks/opentelemetry/grpc.rs | 177 ++++++++++++++++++++++---------- src/sinks/opentelemetry/mod.rs | 7 +- 2 files changed, 125 insertions(+), 59 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 6035425835170..a97acd781bbdf 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -137,23 +137,26 @@ pub struct GrpcSinkConfig { impl GrpcSinkConfig { pub async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { - if self.uri.is_dynamic() { - return Err( - "template syntax is not supported for `uri` with the gRPC transport; \ - use a literal URI (e.g. `http://localhost:4317`)" - .into(), - ); - } + // For static URIs, parse at build time for the healthcheck. + // Dynamic URIs are rendered per-event during sink execution. + let static_uri = if self.uri.is_dynamic() { + None + } else { + Some(with_default_scheme( + self.uri + .get_ref() + .parse() + .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?, + self.tls.is_some(), + )?) + }; - let uri = with_default_scheme( - self.uri - .get_ref() - .parse() - .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?, - self.tls.is_some(), - )?; + let use_https = self.tls.is_some() + || static_uri + .as_ref() + .is_some_and(|u| u.scheme_str() == Some("https")); - let tls = if uri.scheme_str() == Some("https") { + let tls = if use_https { MaybeTlsSettings::tls_client(self.tls.as_ref())? } else { MaybeTlsSettings::Raw(()) @@ -197,11 +200,11 @@ impl GrpcSinkConfig { let client = new_grpc_client(&tls, cx.proxy())?; let healthcheck = Box::pin(grpc_healthcheck( client.clone(), - uri.clone(), + static_uri, static_grpc_headers.clone(), cx.healthcheck, )); - let service = OtlpGrpcService::new(client, uri, use_gzip, static_grpc_headers); + let service = OtlpGrpcService::new(client, use_gzip, static_grpc_headers); let request_settings = self.request.tower.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; @@ -214,6 +217,8 @@ impl GrpcSinkConfig { batch_settings, service, dynamic_header_templates: dynamic_grpc_header_templates, + uri_template: self.uri.clone(), + use_https, }; Ok((VectorSink::from_event_streamsink(sink), healthcheck)) @@ -240,7 +245,7 @@ fn new_grpc_client( async fn grpc_healthcheck( client: hyper::Client>, BoxBody>, - uri: Uri, + uri: Option, headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, options: SinkHealthcheckOptions, ) -> crate::Result<()> { @@ -248,6 +253,11 @@ async fn grpc_healthcheck( return Ok(()); } + let Some(uri) = uri else { + debug!("Skipping gRPC healthcheck: URI is a dynamic template resolved at runtime."); + return Ok(()); + }; + use tonic::Code; use tonic_health::pb::{HealthCheckRequest, health_client::HealthClient}; @@ -326,6 +336,7 @@ pub enum OtlpGrpcError { #[derive(Clone)] pub struct OtlpGrpcRequest { pub signal: OtlpSignalRequest, + pub uri: Uri, /// Per-event rendered values for dynamic (templated) metadata headers. pub dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, pub finalizers: EventFinalizers, @@ -372,50 +383,62 @@ impl DriverResponse for OtlpGrpcResponse { // ── Service ────────────────────────────────────────────────────────────────── +struct CachedClients { + uri: Uri, + logs: LogsServiceClient, + metrics: MetricsServiceClient, + traces: TraceServiceClient, +} + #[derive(Clone)] pub struct OtlpGrpcService { - logs_client: LogsServiceClient, - metrics_client: MetricsServiceClient, - traces_client: TraceServiceClient, + /// Tonic clients for the most-recently-seen URI; rebuilt when the rendered URI changes. + clients: std::sync::Arc>>, + hyper_client: hyper::Client>, BoxBody>, + compression: bool, headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, - protocol: String, - endpoint: String, } impl OtlpGrpcService { pub fn new( hyper_client: hyper::Client>, BoxBody>, - uri: Uri, compression: bool, headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, ) -> Self { - let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(uri.clone()); - - let svc = HyperSvc { - uri, - client: hyper_client, - }; - - let mut logs_client = LogsServiceClient::new(svc.clone()); - let mut metrics_client = MetricsServiceClient::new(svc.clone()); - let mut traces_client = TraceServiceClient::new(svc); - - if compression { - logs_client = logs_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); - metrics_client = - metrics_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); - traces_client = traces_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); - } - Self { - logs_client, - metrics_client, - traces_client, + clients: std::sync::Arc::new(std::sync::Mutex::new(None)), + hyper_client, + compression, headers, - protocol, - endpoint, } } + + /// Returns cloned tonic clients for `uri`, rebuilding them if the URI changed. + /// The mutex is held only during the synchronous rebuild, never across any `.await`. + fn clients_for( + &self, + uri: &Uri, + ) -> ( + LogsServiceClient, + MetricsServiceClient, + TraceServiceClient, + ) { + let mut guard = self.clients.lock().expect("client lock poisoned"); + if guard.as_ref().is_none_or(|c| &c.uri != uri) { + let svc = HyperSvc { uri: uri.clone(), client: self.hyper_client.clone() }; + let mut logs = LogsServiceClient::new(svc.clone()); + let mut metrics = MetricsServiceClient::new(svc.clone()); + let mut traces = TraceServiceClient::new(svc); + if self.compression { + logs = logs.send_compressed(tonic::codec::CompressionEncoding::Gzip); + metrics = metrics.send_compressed(tonic::codec::CompressionEncoding::Gzip); + traces = traces.send_compressed(tonic::codec::CompressionEncoding::Gzip); + } + *guard = Some(CachedClients { uri: uri.clone(), logs, metrics, traces }); + } + let c = guard.as_ref().expect("just populated"); + (c.logs.clone(), c.metrics.clone(), c.traces.clone()) + } } impl Service for OtlpGrpcService { @@ -428,7 +451,11 @@ impl Service for OtlpGrpcService { } fn call(&mut self, mut req: OtlpGrpcRequest) -> Self::Future { - let mut svc = self.clone(); + let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(req.uri.clone()); + // Rebuild clients if the URI changed; clone them out before any `.await`. + let (mut logs_client, mut metrics_client, mut traces_client) = + self.clients_for(&req.uri); + let static_headers = self.headers.clone(); let metadata = std::mem::take(req.metadata_mut()); let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); @@ -437,7 +464,7 @@ impl Service for OtlpGrpcService { ($client:expr, $payload:expr) => {{ let len = $payload.encoded_len(); let mut grpc_req = tonic::Request::new($payload); - for (key, value) in &svc.headers { + for (key, value) in &static_headers { grpc_req.metadata_mut().insert(key.clone(), value.clone()); } for (key, value) in &req.dynamic_headers { @@ -452,15 +479,15 @@ impl Service for OtlpGrpcService { } let byte_size = match req.signal { - OtlpSignalRequest::Logs(r) => export!(svc.logs_client, r), - OtlpSignalRequest::Metrics(r) => export!(svc.metrics_client, r), - OtlpSignalRequest::Traces(r) => export!(svc.traces_client, r), + OtlpSignalRequest::Logs(r) => export!(logs_client, r), + OtlpSignalRequest::Metrics(r) => export!(metrics_client, r), + OtlpSignalRequest::Traces(r) => export!(traces_client, r), }; emit!(EndpointBytesSent { byte_size, - protocol: &svc.protocol, - endpoint: &svc.endpoint, + protocol: &protocol, + endpoint: &endpoint, }); Ok(OtlpGrpcResponse { events_byte_size }) @@ -509,6 +536,7 @@ struct OtlpEventData { json_byte_size: GroupedCountByteSize, finalizers: EventFinalizers, signal: OtlpSignal, + uri: Uri, dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, } @@ -546,7 +574,8 @@ struct OtlpBatch { logs: Option>, metrics: Option>, traces: Option>, - /// Dynamic header values captured from the first event in the batch. + /// URI and dynamic header values captured from the first event in the batch. + uri: Option, dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, } @@ -561,6 +590,8 @@ impl Clone for OtlpBatch { pub struct OtlpGrpcSink { pub batch_settings: BatcherSettings, pub service: S, + uri_template: Template, + use_https: bool, dynamic_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)>, } @@ -576,10 +607,38 @@ where error!("Failed to create OTLP serializer: {}", e); })?; + let uri_template = self.uri_template.clone(); + let use_https = self.use_https; let dynamic_header_templates = self.dynamic_header_templates.clone(); input .filter_map(move |mut event| { + let uri = match uri_template.render_string(&event) { + Ok(rendered) => match rendered.parse::() { + Ok(parsed) => match with_default_scheme(parsed, use_https) { + Ok(u) => u, + Err(e) => { + emit!(crate::internal_events::SinkRequestBuildError { + error: format!("invalid gRPC URI after rendering template: {e}"), + }); + return futures::future::ready(None); + } + }, + Err(e) => { + emit!(crate::internal_events::SinkRequestBuildError { + error: format!("failed to parse rendered gRPC URI: {e}"), + }); + return futures::future::ready(None); + } + }, + Err(e) => { + emit!(crate::internal_events::SinkRequestBuildError { + error: format!("failed to render gRPC URI template: {e}"), + }); + return futures::future::ready(None); + } + }; + let dynamic_headers: Vec<_> = dynamic_header_templates .iter() .filter_map(|(key, template)| { @@ -608,6 +667,7 @@ where json_byte_size, finalizers: event.take_finalizers(), signal, + uri, dynamic_headers, }; futures::future::ready(Some(data)) @@ -628,7 +688,8 @@ where .batched(self.batch_settings.as_reducer_config( |data: &OtlpEventData| data.signal.encoded_len(), BatchReduce::new(|batch: &mut OtlpBatch, item: OtlpEventData| { - if batch.dynamic_headers.is_empty() { + if batch.uri.is_none() { + batch.uri = Some(item.uri); batch.dynamic_headers = item.dynamic_headers; } @@ -668,6 +729,9 @@ where }), )) .flat_map(|batch| { + let Some(uri) = batch.uri else { + return futures::stream::iter(Vec::new()); + }; let mut requests = Vec::new(); let dynamic_headers = batch.dynamic_headers; @@ -684,6 +748,7 @@ where ); requests.push(OtlpGrpcRequest { signal: OtlpSignalRequest::$variant(data.request), + uri: uri.clone(), dynamic_headers: dynamic_headers.clone(), finalizers: data.finalizers, metadata: builder.with_request_size(bytes_len), diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 18800c31abe28..35eda0502aa36 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -78,10 +78,11 @@ pub struct OpenTelemetryConfig { /// The URI to send requests to. /// - /// Must include a scheme (`http://` or `https://`) and a port. + /// Supports template syntax (e.g. `http://{{ host }}:4317`). Must include a scheme + /// (`http://` or `https://`) and a port. /// - /// For the HTTP transport, template syntax is supported (e.g. `http://{{ host }}:4318/v1/logs`). - /// The gRPC transport requires a literal URI; template syntax is not supported. + /// For the gRPC transport, the template is rendered once per batch using the first event + /// in the batch. /// /// # Examples /// From f29aa97170505bf0c853c8abe8cf8a6248c67c94 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 10:40:58 -0400 Subject: [PATCH 21/68] test(opentelemetry sink): add integration test for gRPC template URI --- src/sinks/opentelemetry/integration_tests.rs | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs index e6dbf470f4cad..aa29f5d0d5457 100644 --- a/src/sinks/opentelemetry/integration_tests.rs +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -24,6 +24,14 @@ fn sink_grpc_address() -> String { .unwrap_or_else(|_| "opentelemetry-collector:4317".to_owned()) } +fn otlp_log_event_with_host(host: &str) -> vector_lib::event::Event { + let mut event = otlp_log_event(); + if let vector_lib::event::Event::Log(ref mut log) = event { + log.insert("host", host.to_owned()); + } + event +} + fn otlp_log_event() -> vector_lib::event::Event { let req = ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { @@ -74,3 +82,20 @@ async fn delivers_logs_via_grpc() { let events = vec![otlp_log_event()]; run_and_assert_sink_compliance(sink, stream::iter(events), &HTTP_SINK_TAGS).await; } + +#[tokio::test] +async fn delivers_logs_via_grpc_template_uri() { + let host = sink_grpc_address(); + wait_for_tcp(host.clone()).await; + + let config: GrpcSinkConfig = toml::from_str(r#" + uri = "http://{{ host }}:4317" + "#) + .unwrap(); + + let (sink, _healthcheck) = config.build(SinkContext::default()).await.unwrap(); + + // The event carries `host` so the template renders to the collector address. + let events = vec![otlp_log_event_with_host(host.split(':').next().unwrap())]; + run_and_assert_sink_compliance(sink, stream::iter(events), &HTTP_SINK_TAGS).await; +} From 90b194be0a5ee70c30311c80c6ec8c8c77f6434e Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 11:57:44 -0400 Subject: [PATCH 22/68] Restrict grpc to only support logs and traces --- src/sinks/opentelemetry/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 35eda0502aa36..52f64b7ab3982 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -9,7 +9,7 @@ use vector_lib::configurable::configurable_component; use crate::{ codecs::EncodingConfigWithFraming, - config::{AcknowledgementsConfig, Input, SinkConfig, SinkContext}, + config::{AcknowledgementsConfig, DataType, Input, SinkConfig, SinkContext}, http::Auth, sinks::{ Healthcheck, VectorSink, @@ -176,7 +176,7 @@ impl SinkConfig for OpenTelemetryConfig { fn input(&self) -> Input { match &self.protocol { OtlpProtocol::Http { encoding, .. } => Input::new(encoding.config().1.input_type()), - OtlpProtocol::Grpc { .. } => Input::all(), + OtlpProtocol::Grpc { .. } => Input::new(DataType::Log | DataType::Trace), } } From 1af1b93c33082918c54d7cbeff1807457901be1c Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 14:14:56 -0400 Subject: [PATCH 23/68] fix(opentelemetry sink): partition gRPC batches by rendered URI and headers --- src/sinks/opentelemetry/grpc.rs | 192 +++++++++++++++++++++++--------- 1 file changed, 137 insertions(+), 55 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index a97acd781bbdf..5191d53f2d786 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -32,7 +32,8 @@ use vector_lib::{ }, }, request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, - stream::{BatcherSettings, DriverResponse, batcher::data::BatchReduce}, + partition::Partitioner, + stream::{BatcherSettings, DriverResponse}, }; use crate::{ @@ -530,6 +531,37 @@ impl Service> for HyperSvc { // ── Sink ───────────────────────────────────────────────────────────────────── +/// Partition key for gRPC batches. Events that render to different URIs or +/// dynamic headers must be sent in separate requests, so they are batched +/// independently. The key stores rendered values as strings so it can derive +/// `Hash + Eq` without requiring those traits from `Uri` or tonic types. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct BatchPartitionKey { + uri: String, + /// Dynamic header (key, rendered-value) pairs, sorted for deterministic equality. + headers: Vec<(String, String)>, +} + +struct GrpcPartitioner; + +impl Partitioner for GrpcPartitioner { + type Item = OtlpEventData; + type Key = BatchPartitionKey; + + fn partition(&self, item: &OtlpEventData) -> BatchPartitionKey { + let mut headers: Vec<(String, String)> = item + .dynamic_headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_owned())) + .collect(); + headers.sort_unstable(); + BatchPartitionKey { + uri: item.uri.to_string(), + headers, + } + } +} + /// Intermediate event data extracted before batching. struct OtlpEventData { byte_size: usize, @@ -540,6 +572,16 @@ struct OtlpEventData { dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, } +impl ByteSizeOf for OtlpEventData { + fn size_of(&self) -> usize { + std::mem::size_of::() + self.allocated_bytes() + } + + fn allocated_bytes(&self) -> usize { + self.signal.encoded_len() + } +} + /// Pre-decoded OTLP signal for a single event. enum OtlpSignal { Logs(ExportLogsServiceRequest), @@ -574,16 +616,65 @@ struct OtlpBatch { logs: Option>, metrics: Option>, traces: Option>, - /// URI and dynamic header values captured from the first event in the batch. - uri: Option, - dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, } -impl Clone for OtlpBatch { - fn clone(&self) -> Self { - // OtlpBatch is used only in the batcher accumulator; Clone is required by the - // BatchReduce API but the accumulator is always the initial default value. - Self::default() +impl OtlpBatch { + fn push(&mut self, item: OtlpEventData) { + let OtlpEventData { + byte_size, + json_byte_size, + finalizers, + signal, + uri: _, + dynamic_headers: _, + } = item; + + macro_rules! accumulate { + ($field:ident, $req:expr, $merge:expr) => { + match &mut self.$field { + Some(existing) => { + $merge(&mut existing.request, $req); + existing.finalizers.merge(finalizers); + existing.event_count += 1; + existing.byte_size += byte_size; + existing.json_byte_size += json_byte_size; + } + slot => { + *slot = Some(SignalData { + request: $req, + finalizers, + event_count: 1, + byte_size, + json_byte_size, + }); + } + } + }; + } + + match signal { + OtlpSignal::Logs(req) => accumulate!( + logs, + req, + |e: &mut ExportLogsServiceRequest, r: ExportLogsServiceRequest| { + e.resource_logs.extend(r.resource_logs) + } + ), + OtlpSignal::Metrics(req) => accumulate!( + metrics, + req, + |e: &mut ExportMetricsServiceRequest, r: ExportMetricsServiceRequest| { + e.resource_metrics.extend(r.resource_metrics) + } + ), + OtlpSignal::Traces(req) => accumulate!( + traces, + req, + |e: &mut ExportTraceServiceRequest, r: ExportTraceServiceRequest| { + e.resource_spans.extend(r.resource_spans) + } + ), + } } } @@ -685,55 +776,46 @@ where } } }) - .batched(self.batch_settings.as_reducer_config( - |data: &OtlpEventData| data.signal.encoded_len(), - BatchReduce::new(|batch: &mut OtlpBatch, item: OtlpEventData| { - if batch.uri.is_none() { - batch.uri = Some(item.uri); - batch.dynamic_headers = item.dynamic_headers; - } - - macro_rules! accumulate { - ($field:ident, $req:expr, $merge:expr) => { - match &mut batch.$field { - Some(existing) => { - $merge(&mut existing.request, $req); - existing.finalizers.merge(item.finalizers); - existing.event_count += 1; - existing.byte_size += item.byte_size; - existing.json_byte_size += item.json_byte_size; - } - slot => { - *slot = Some(SignalData { - request: $req, - finalizers: item.finalizers, - event_count: 1, - byte_size: item.byte_size, - json_byte_size: item.json_byte_size, - }); - } - } - }; - } - match item.signal { - OtlpSignal::Logs(req) => accumulate!(logs, req, |e: &mut ExportLogsServiceRequest, r: ExportLogsServiceRequest| { - e.resource_logs.extend(r.resource_logs) - }), - OtlpSignal::Metrics(req) => accumulate!(metrics, req, |e: &mut ExportMetricsServiceRequest, r: ExportMetricsServiceRequest| { - e.resource_metrics.extend(r.resource_metrics) - }), - OtlpSignal::Traces(req) => accumulate!(traces, req, |e: &mut ExportTraceServiceRequest, r: ExportTraceServiceRequest| { - e.resource_spans.extend(r.resource_spans) - }), + // Partition by (rendered URI, dynamic headers) so that events destined for + // different collectors or with different tenant metadata are never merged into + // the same batch. Each partition is independently flushed by size or timeout. + .batched_partitioned( + GrpcPartitioner, + self.batch_settings.timeout, + |_| self.batch_settings.as_byte_size_config(), + ) + .flat_map(|(key, items)| { + // Re-parse the URI from the partition key (always succeeds: it was validated + // in filter_map before being stored in the key). + let uri: Uri = match key.uri.parse::() { + Ok(u) => u, + Err(e) => { + error!("Failed to re-parse gRPC batch partition URI: {e}"); + return futures::stream::iter(Vec::new()); } - }), - )) - .flat_map(|batch| { - let Some(uri) = batch.uri else { - return futures::stream::iter(Vec::new()); }; + + let dynamic_headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )> = key + .headers + .iter() + .filter_map(|(k, v)| { + Some(( + tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()).ok()?, + tonic::metadata::AsciiMetadataValue::try_from(v.as_str()).ok()?, + )) + }) + .collect(); + + // Reduce the partitioned items into per-signal accumulators. + let mut batch = OtlpBatch::default(); + for item in items { + batch.push(item); + } + let mut requests = Vec::new(); - let dynamic_headers = batch.dynamic_headers; macro_rules! push_signal { ($field:ident, $variant:ident) => { From 03bdf873d5dbe4de6695bb09393c34ef41adf570 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 14:17:33 -0400 Subject: [PATCH 24/68] fix(opentelemetry sink): enable TLS for templated HTTPS gRPC URIs --- src/sinks/opentelemetry/grpc.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 5191d53f2d786..cb79f4bda2431 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -152,10 +152,15 @@ impl GrpcSinkConfig { )?) }; + // For dynamic templates like `https://{{ host }}:4317` the static_uri is None, so + // we also check whether the literal prefix of the template string is "https://". + // This covers the common case where the scheme is a fixed literal even though the + // host/port are templated. let use_https = self.tls.is_some() || static_uri .as_ref() - .is_some_and(|u| u.scheme_str() == Some("https")); + .is_some_and(|u| u.scheme_str() == Some("https")) + || self.uri.get_ref().starts_with("https://"); let tls = if use_https { MaybeTlsSettings::tls_client(self.tls.as_ref())? From 9f2627b4e7d6bc2177e484a63c9a81311913119c Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 14:59:31 -0400 Subject: [PATCH 25/68] docs(opentelemetry sink): warn that dynamic URIs require trusted event sources --- src/sinks/opentelemetry/grpc.rs | 1 + src/sinks/opentelemetry/mod.rs | 1 + ...1-pending-p1-ssrf-dynamic-uri-rendering.md | 57 +++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 todos/001-pending-p1-ssrf-dynamic-uri-rendering.md diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index cb79f4bda2431..b22c486588455 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -109,6 +109,7 @@ pub struct GrpcSinkConfig { /// - `http://localhost:4317` /// - `https://otelcol.example.com:4317` #[configurable(metadata(docs::examples = "http://localhost:4317"))] + #[configurable(metadata(docs::warnings = "When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations."))] pub uri: Template, #[configurable(derived)] diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 52f64b7ab3982..8f91b17800a87 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -90,6 +90,7 @@ pub struct OpenTelemetryConfig { /// - `http://localhost:4317` (gRPC) #[configurable(metadata(docs::examples = "http://localhost:5318/v1/logs"))] #[configurable(metadata(docs::examples = "http://localhost:4317"))] + #[configurable(metadata(docs::warnings = "When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations."))] pub uri: Template, #[configurable(derived)] diff --git a/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md b/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md new file mode 100644 index 0000000000000..4f4a13f7a0930 --- /dev/null +++ b/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md @@ -0,0 +1,57 @@ +--- +name: SSRF via unvalidated dynamic URI rendering +description: Dynamic gRPC URI rendered from untrusted event fields with no allowlist or scheme restriction, enabling SSRF +type: finding +status: resolved +priority: p1 +issue_id: "001" +tags: [code-review, security, opentelemetry, grpc] +--- + +## Problem Statement + +When `uri` is a template (e.g. `http://{{ host }}:4317`), the rendered value is taken verbatim from the event field and only validated as a syntactically valid `Uri`. There is no allowlist, no scheme restriction, and no hostname restriction applied to the rendered result. + +An attacker who controls event data (e.g. via a log ingestion pipeline that accepts external input) can inject any value into the `host` field, directing Vector to make gRPC TCP connections to arbitrary internal network hosts including cloud metadata endpoints (`169.254.169.254`), internal Kubernetes services, etc. + +The same primitive applies to dynamic header values — while constrained to ASCII, they are not restricted beyond that type check. + +## Findings + +- **File**: `src/sinks/opentelemetry/grpc.rs` lines 711-736 +- **Mechanism**: `uri_template.render_string(&event)` → `rendered.parse::()` → used directly for connection +- **Documented use case**: per-tenant routing with `{{ host }}` — explicitly designed for scenarios where event data controls egress destination +- **No allowlist or scheme restriction in the URI validation path** + +## Proposed Solutions + +### Option A: URI allowlist config key +Add `allowed_uri_prefixes: Vec` to `GrpcSinkConfig`. Any rendered URI that doesn't match a prefix is dropped with `SinkRequestBuildError`. Fail closed. +- **Pros**: Flexible, operator-controlled, explicit +- **Cons**: Breaking addition to config, operators must configure it to use dynamic URIs +- **Effort**: Medium +- **Risk**: Low (additive) + +### Option B: Restrict dynamic URI to scheme + host changes only +Parse the rendered URI and validate scheme is `http` or `https` only, and optionally restrict port range. Reject file://, ftp://, etc. +- **Pros**: Simpler than allowlist, catches obvious SSRF vectors +- **Cons**: Doesn't prevent SSRF to internal network; `http://169.254.169.254:80` still valid +- **Effort**: Small +- **Risk**: Low + +### Option C: Document as operator responsibility, add security warning to config docs +Add a `#[configurable(metadata(docs::warnings = "..."))]` annotation noting that dynamic URI templates should only be used with trusted event data. +- **Pros**: No code change +- **Cons**: Does not prevent the vulnerability; just warns +- **Effort**: Small +- **Risk**: None + +## Acceptance Criteria + +- [x] Dynamic URI rendering is either restricted (allowlist) or clearly documented as requiring trusted input +- [ ] At minimum, scheme is validated to be `http` or `https` only for rendered URIs +- [ ] Any rendered URI that fails validation emits `SinkRequestBuildError` and drops the event + +## Work Log + +- 2026-03-26: Identified by security-sentinel review agent From fad896facef4a672e2377f8f1fe5bbd71d7bfef5 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 15:01:49 -0400 Subject: [PATCH 26/68] fix(opentelemetry sink): reject rendered gRPC URIs with non-http/https scheme --- src/sinks/opentelemetry/grpc.rs | 15 ++++++++++++++- .../001-pending-p1-ssrf-dynamic-uri-rendering.md | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index b22c486588455..97f75968f0c66 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -713,7 +713,20 @@ where let uri = match uri_template.render_string(&event) { Ok(rendered) => match rendered.parse::() { Ok(parsed) => match with_default_scheme(parsed, use_https) { - Ok(u) => u, + Ok(u) => { + match u.scheme_str() { + Some("http") | Some("https") => u, + other => { + emit!(crate::internal_events::SinkRequestBuildError { + error: format!( + "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", + other.unwrap_or("") + ), + }); + return futures::future::ready(None); + } + } + } Err(e) => { emit!(crate::internal_events::SinkRequestBuildError { error: format!("invalid gRPC URI after rendering template: {e}"), diff --git a/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md b/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md index 4f4a13f7a0930..987e13d17d6a9 100644 --- a/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md +++ b/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md @@ -49,8 +49,8 @@ Add a `#[configurable(metadata(docs::warnings = "..."))]` annotation noting that ## Acceptance Criteria - [x] Dynamic URI rendering is either restricted (allowlist) or clearly documented as requiring trusted input -- [ ] At minimum, scheme is validated to be `http` or `https` only for rendered URIs -- [ ] Any rendered URI that fails validation emits `SinkRequestBuildError` and drops the event +- [x] At minimum, scheme is validated to be `http` or `https` only for rendered URIs +- [x] Any rendered URI that fails validation emits `SinkRequestBuildError` and drops the event ## Work Log From 2afefd23582414ed081476b543fa3d1e69083737 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 15:05:30 -0400 Subject: [PATCH 27/68] fix(opentelemetry sink): drop events whose rendered gRPC URI is https but TLS is not configured --- src/sinks/opentelemetry/grpc.rs | 15 +++++ ...pending-p1-tls-mode-fixed-at-build-time.md | 61 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 todos/002-pending-p1-tls-mode-fixed-at-build-time.md diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 97f75968f0c66..f809cfa1cc9dd 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -715,6 +715,21 @@ where Ok(parsed) => match with_default_scheme(parsed, use_https) { Ok(u) => { match u.scheme_str() { + Some("https") if !use_https => { + // The Hyper client was built without TLS (use_https=false), + // so it cannot complete a TLS handshake. Sending data to an + // https:// endpoint over a plaintext connector would either + // fail or silently transmit unencrypted. Drop the event and + // surface a clear error so the operator can add `tls:` or + // use a static `https://` scheme prefix. + emit!(crate::internal_events::SinkRequestBuildError { + error: "rendered gRPC URI uses \"https\" but the sink \ + has no TLS connector; add a `tls:` block or use \ + a static \"https://\" URI prefix so TLS is \ + enabled at startup", + }); + return futures::future::ready(None); + } Some("http") | Some("https") => u, other => { emit!(crate::internal_events::SinkRequestBuildError { diff --git a/todos/002-pending-p1-tls-mode-fixed-at-build-time.md b/todos/002-pending-p1-tls-mode-fixed-at-build-time.md new file mode 100644 index 0000000000000..3598f1ca63149 --- /dev/null +++ b/todos/002-pending-p1-tls-mode-fixed-at-build-time.md @@ -0,0 +1,61 @@ +--- +name: TLS mode fixed at build time silently sends plaintext for some dynamic URI templates +description: use_https flag computed once at build time; templates like {{ scheme }}://{{ host }} may silently connect over plaintext +type: finding +status: resolved +priority: p1 +issue_id: "002" +tags: [code-review, security, tls, opentelemetry, grpc] +--- + +## Problem Statement + +The `use_https` flag is computed once at sink build time: + +```rust +let use_https = self.tls.is_some() + || static_uri.as_ref().is_some_and(|u| u.scheme_str() == Some("https")) + || self.uri.get_ref().starts_with("https://"); +``` + +The third check (`starts_with("https://")`) only matches when the literal template string begins with `https://`. A template like `{{ scheme }}://{{ host }}:4317` or even `http://{{ host }}:4317` where an event renders to `https://...` will have `use_https = false`. The underlying Hyper client is then built with `MaybeTlsSettings::Raw(())` — a plaintext connector — regardless of the rendered URI scheme. + +This means operators using fully-dynamic URI templates who believe they have TLS will silently send telemetry in plaintext. + +## Findings + +- **File**: `src/sinks/opentelemetry/grpc.rs` lines 155-169 +- **Risk**: Silent plaintext egress for PII/confidential telemetry +- **Affected configurations**: Any dynamic URI template where the scheme is not a literal static prefix + +## Proposed Solutions + +### Option A: Validate rendered URI scheme matches build-time TLS mode +At event render time, check that the rendered URI's scheme matches `use_https`. If they conflict, drop the event with a `SinkRequestBuildError` and a clear message. +- **Pros**: Fails loudly, no silent plaintext +- **Cons**: Drops events when config is ambiguous; operator must configure `tls:` block or use static scheme +- **Effort**: Small +- **Risk**: Low + +### Option B: Require scheme to be a static literal in dynamic URI templates +At config parse time, validate that the scheme portion of any template URI is not itself a template expression. Reject configs like `{{ scheme }}://{{ host }}`. +- **Pros**: Eliminates the ambiguity at startup +- **Cons**: Restricts valid template patterns +- **Effort**: Medium (requires template parsing) +- **Risk**: Low + +### Option C: Document the limitation clearly in config schema +Add a docstring note to the `uri` field: "When using a dynamic template that renders to `https://`, you must also configure `tls:` to ensure TLS is used. The sink cannot infer TLS from a fully-dynamic scheme." +- **Pros**: No code change, low effort +- **Cons**: Doesn't prevent the issue, just warns +- **Effort**: Small +- **Risk**: None + +## Acceptance Criteria + +- [x] Operators are not silently sent over plaintext when they configure a dynamic `https://` URI template +- [x] Either the scheme mismatch is detected and rejected at runtime, or config validation prevents it + +## Work Log + +- 2026-03-26: Identified by security-sentinel review agent From fea069abcb7d49f3b1d4ad0bef983a9d23f4a3ce Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 16:08:36 -0400 Subject: [PATCH 28/68] fix(opentelemetry sink): emit ComponentEventsDropped, drop events on header render failure, simplify gRPC internals --- src/sinks/opentelemetry/grpc.rs | 276 ++++++++++-------- src/sinks/opentelemetry/integration_tests.rs | 8 +- src/sinks/opentelemetry/mod.rs | 4 +- .../sinks/generated/opentelemetry.cue | 8 +- 4 files changed, 162 insertions(+), 134 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index f809cfa1cc9dd..e7dbaf41c771f 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -7,7 +7,10 @@ use std::{ use async_trait::async_trait; use bytes::BytesMut; use futures::{StreamExt, TryFutureExt, future::BoxFuture, stream::BoxStream}; -use http::Uri; +use http::{ + Uri, + uri::{PathAndQuery, Scheme}, +}; use hyper::client::HttpConnector; use hyper_openssl::HttpsConnector; use hyper_proxy::ProxyConnector; @@ -21,6 +24,7 @@ use vector_lib::{ codecs::encoding::format::OtlpSerializer, config::telemetry, configurable::configurable_component, + internal_event::{ComponentEventsDropped, UNINTENTIONAL}, opentelemetry::proto::{ RESOURCE_LOGS_JSON_FIELD, RESOURCE_METRICS_JSON_FIELD, RESOURCE_SPANS_JSON_FIELD, collector::{ @@ -31,8 +35,8 @@ use vector_lib::{ trace::v1::{ExportTraceServiceRequest, trace_service_client::TraceServiceClient}, }, }, - request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, partition::Partitioner, + request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, stream::{BatcherSettings, DriverResponse}, }; @@ -70,24 +74,9 @@ pub enum GrpcCompression { pub(super) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { if uri.scheme().is_none() { let mut parts = uri.into_parts(); - parts.scheme = if tls { - Some( - "https" - .parse() - .unwrap_or_else(|_| unreachable!("https should be valid")), - ) - } else { - Some( - "http" - .parse() - .unwrap_or_else(|_| unreachable!("http should be valid")), - ) - }; + parts.scheme = Some(if tls { Scheme::HTTPS } else { Scheme::HTTP }); if parts.path_and_query.is_none() { - parts.path_and_query = Some( - "/".parse() - .unwrap_or_else(|_| unreachable!("root should be valid")), - ); + parts.path_and_query = Some(PathAndQuery::from_static("/")); } Ok(Uri::from_parts(parts)?) } else { @@ -109,7 +98,9 @@ pub struct GrpcSinkConfig { /// - `http://localhost:4317` /// - `https://otelcol.example.com:4317` #[configurable(metadata(docs::examples = "http://localhost:4317"))] - #[configurable(metadata(docs::warnings = "When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations."))] + #[configurable(metadata( + docs::warnings = "When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations." + ))] pub uri: Template, #[configurable(derived)] @@ -175,8 +166,7 @@ impl GrpcSinkConfig { // Static headers are pre-parsed once and used for the healthcheck and every export. // Dynamic headers are rendered per-event so that templated fields (e.g. tenant IDs) // resolve correctly at export time. - let (static_header_strings, dynamic_header_templates_raw) = - self.request.split_headers(); + let (static_header_strings, dynamic_header_templates_raw) = self.request.split_headers(); let parse_key = |k: &str| { tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) @@ -253,7 +243,10 @@ fn new_grpc_client( async fn grpc_healthcheck( client: hyper::Client>, BoxBody>, uri: Option, - headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, + headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, options: SinkHealthcheckOptions, ) -> crate::Result<()> { if !options.enabled { @@ -278,8 +271,7 @@ async fn grpc_healthcheck( req.metadata_mut().insert(key, value); } - match health_client.check(req).await - { + match health_client.check(req).await { Ok(response) => { use tonic_health::pb::health_check_response::ServingStatus; let status = response.into_inner().status; @@ -289,7 +281,7 @@ async fn grpc_healthcheck( Err(format!("gRPC collector reported non-serving status: {status}").into()) } } - // Server is reachable but does not implement the health protocol — treat as healthy. + // Server is reachable but does not implement the health protocol; treat as healthy. Err(status) if status.code() == Code::Unimplemented => Ok(()), Err(status) => Err(Box::new(OtlpGrpcError::Request { source: status })), } @@ -320,6 +312,10 @@ impl RetryLogic for OtlpGrpcRetryLogic { | OutOfRange | Unimplemented | Unauthenticated + // DataLoss: per gRPC spec this means unrecoverable data corruption, not + // retriable. Note that the OTLP partial-success model can also surface + // DataLoss for a partial write; we treat it as non-retriable (consistent with + // the vector sink) to avoid sending an already-accepted partial batch twice. | DataLoss ), } @@ -338,26 +334,21 @@ pub enum OtlpGrpcError { // ── Request/Response ───────────────────────────────────────────────────────── /// A single-signal OTLP export request. One request per signal type ensures -/// that retries are atomic — a failed metrics export cannot duplicate a +/// that retries are atomic: a failed metrics export cannot duplicate a /// previously-accepted logs export. #[derive(Clone)] pub struct OtlpGrpcRequest { - pub signal: OtlpSignalRequest, + pub signal: OtlpSignal, pub uri: Uri, /// Per-event rendered values for dynamic (templated) metadata headers. - pub dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, + pub dynamic_headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, } -/// The OTLP export payload for a single signal type. -#[derive(Clone)] -pub enum OtlpSignalRequest { - Logs(ExportLogsServiceRequest), - Metrics(ExportMetricsServiceRequest), - Traces(ExportTraceServiceRequest), -} - impl Finalizable for OtlpGrpcRequest { fn take_finalizers(&mut self) -> EventFinalizers { self.finalizers.take_finalizers() @@ -403,14 +394,20 @@ pub struct OtlpGrpcService { clients: std::sync::Arc>>, hyper_client: hyper::Client>, BoxBody>, compression: bool, - headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, + headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, } impl OtlpGrpcService { pub fn new( hyper_client: hyper::Client>, BoxBody>, compression: bool, - headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, + headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, ) -> Self { Self { clients: std::sync::Arc::new(std::sync::Mutex::new(None)), @@ -432,7 +429,10 @@ impl OtlpGrpcService { ) { let mut guard = self.clients.lock().expect("client lock poisoned"); if guard.as_ref().is_none_or(|c| &c.uri != uri) { - let svc = HyperSvc { uri: uri.clone(), client: self.hyper_client.clone() }; + let svc = HyperSvc { + uri: uri.clone(), + client: self.hyper_client.clone(), + }; let mut logs = LogsServiceClient::new(svc.clone()); let mut metrics = MetricsServiceClient::new(svc.clone()); let mut traces = TraceServiceClient::new(svc); @@ -441,7 +441,12 @@ impl OtlpGrpcService { metrics = metrics.send_compressed(tonic::codec::CompressionEncoding::Gzip); traces = traces.send_compressed(tonic::codec::CompressionEncoding::Gzip); } - *guard = Some(CachedClients { uri: uri.clone(), logs, metrics, traces }); + *guard = Some(CachedClients { + uri: uri.clone(), + logs, + metrics, + traces, + }); } let c = guard.as_ref().expect("just populated"); (c.logs.clone(), c.metrics.clone(), c.traces.clone()) @@ -460,8 +465,7 @@ impl Service for OtlpGrpcService { fn call(&mut self, mut req: OtlpGrpcRequest) -> Self::Future { let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(req.uri.clone()); // Rebuild clients if the URI changed; clone them out before any `.await`. - let (mut logs_client, mut metrics_client, mut traces_client) = - self.clients_for(&req.uri); + let (mut logs_client, mut metrics_client, mut traces_client) = self.clients_for(&req.uri); let static_headers = self.headers.clone(); let metadata = std::mem::take(req.metadata_mut()); let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); @@ -486,9 +490,9 @@ impl Service for OtlpGrpcService { } let byte_size = match req.signal { - OtlpSignalRequest::Logs(r) => export!(logs_client, r), - OtlpSignalRequest::Metrics(r) => export!(metrics_client, r), - OtlpSignalRequest::Traces(r) => export!(traces_client, r), + OtlpSignal::Logs(r) => export!(logs_client, r), + OtlpSignal::Metrics(r) => export!(metrics_client, r), + OtlpSignal::Traces(r) => export!(traces_client, r), }; emit!(EndpointBytesSent { @@ -522,12 +526,24 @@ impl Service> for HyperSvc { } fn call(&mut self, mut req: hyper::Request) -> Self::Future { + // SAFETY: `self.uri` is always produced by `with_default_scheme`, which guarantees + // a scheme and a path_and_query. Tonic always sets a path on the request URI. let uri = Uri::builder() - .scheme(self.uri.scheme().unwrap().clone()) - .authority(self.uri.authority().unwrap().clone()) - .path_and_query(req.uri().path_and_query().unwrap().clone()) + .scheme(self.uri.scheme().expect("uri always has a scheme").clone()) + .authority( + self.uri + .authority() + .expect("uri always has an authority") + .clone(), + ) + .path_and_query( + req.uri() + .path_and_query() + .expect("tonic request always has a path") + .clone(), + ) .build() - .unwrap(); + .expect("uri components are always valid"); *req.uri_mut() = uri; @@ -575,7 +591,10 @@ struct OtlpEventData { finalizers: EventFinalizers, signal: OtlpSignal, uri: Uri, - dynamic_headers: Vec<(tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue)>, + dynamic_headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, } impl ByteSizeOf for OtlpEventData { @@ -588,8 +607,11 @@ impl ByteSizeOf for OtlpEventData { } } -/// Pre-decoded OTLP signal for a single event. -enum OtlpSignal { +/// OTLP signal payload. Used as per-event intermediate data before batching and as the +/// request payload sent via gRPC after batching. Each variant corresponds to one signal +/// type so that requests can be retried independently. +#[derive(Clone)] +pub enum OtlpSignal { Logs(ExportLogsServiceRequest), Metrics(ExportMetricsServiceRequest), Traces(ExportTraceServiceRequest), @@ -710,6 +732,20 @@ where input .filter_map(move |mut event| { + macro_rules! drop_event { + ($reason:expr) => {{ + let reason_owned: String = $reason.to_string(); + emit!(crate::internal_events::SinkRequestBuildError { + error: reason_owned.as_str() + }); + emit!(ComponentEventsDropped:: { + count: 1, + reason: reason_owned.as_str(), + }); + return futures::future::ready(None); + }}; + } + let uri = match uri_template.render_string(&event) { Ok(rendered) => match rendered.parse::() { Ok(parsed) => match with_default_scheme(parsed, use_https) { @@ -722,66 +758,65 @@ where // fail or silently transmit unencrypted. Drop the event and // surface a clear error so the operator can add `tls:` or // use a static `https://` scheme prefix. - emit!(crate::internal_events::SinkRequestBuildError { - error: "rendered gRPC URI uses \"https\" but the sink \ - has no TLS connector; add a `tls:` block or use \ - a static \"https://\" URI prefix so TLS is \ - enabled at startup", - }); - return futures::future::ready(None); + drop_event!( + "rendered gRPC URI uses \"https\" but the sink \ + has no TLS connector; add a `tls:` block or use \ + a static \"https://\" URI prefix so TLS is \ + enabled at startup" + ); } Some("http") | Some("https") => u, other => { - emit!(crate::internal_events::SinkRequestBuildError { - error: format!( - "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", - other.unwrap_or("") - ), - }); - return futures::future::ready(None); + drop_event!(format!( + "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", + other.unwrap_or("") + )); } } } Err(e) => { - emit!(crate::internal_events::SinkRequestBuildError { - error: format!("invalid gRPC URI after rendering template: {e}"), - }); - return futures::future::ready(None); + drop_event!(format!("invalid gRPC URI after rendering template: {e}")); } }, Err(e) => { - emit!(crate::internal_events::SinkRequestBuildError { - error: format!("failed to parse rendered gRPC URI: {e}"), - }); - return futures::future::ready(None); + drop_event!(format!("failed to parse rendered gRPC URI: {e}")); } }, Err(e) => { - emit!(crate::internal_events::SinkRequestBuildError { - error: format!("failed to render gRPC URI template: {e}"), - }); - return futures::future::ready(None); + drop_event!(format!("failed to render gRPC URI template: {e}")); } }; - let dynamic_headers: Vec<_> = dynamic_header_templates - .iter() - .filter_map(|(key, template)| { - let rendered = template - .render_string(&event) - .map_err(|e| { - warn!("Failed to render gRPC metadata template for key {:?}: {e}", key.as_str()); - }) - .ok()?; - let value = tonic::metadata::AsciiMetadataValue::try_from(rendered.as_str()) - .map_err(|e| { - warn!("Rendered gRPC metadata value for key {:?} is not valid ASCII: {e}", key.as_str()); - }) - .ok()?; - Some((key.clone(), value)) - }) - .collect(); + // Render dynamic headers. If any required header fails to render or produces a + // non-ASCII value, drop the entire event rather than forwarding without it. + // Silently omitting a header (e.g. X-Tenant-ID) could bypass authorization on + // the receiving collector. + let mut dynamic_headers = Vec::with_capacity(dynamic_header_templates.len()); + for (key, template) in &dynamic_header_templates { + match template.render_string(&event) { + Ok(rendered) => { + match tonic::metadata::AsciiMetadataValue::try_from(rendered.as_str()) { + Ok(value) => dynamic_headers.push((key.clone(), value)), + Err(e) => { + drop_event!(format!( + "gRPC metadata value for key {:?} is not valid ASCII: {e}", + key.as_str() + )); + } + } + } + Err(e) => { + drop_event!(format!( + "failed to render gRPC metadata template for key {:?}: {e}", + key.as_str() + )); + } + } + } + // TODO(perf): OtlpSerializer only exposes a byte-level Encoder interface, so + // encode_event must encode to bytes and then decode back to a typed proto struct. + // Exposing a typed output from OtlpSerializer would eliminate this roundtrip. let signal = encode_event(&mut serializer, &event); match signal { Ok(Some(signal)) => { @@ -798,15 +833,17 @@ where futures::future::ready(Some(data)) } Ok(None) => { - // Event type not supported (e.g. native Vector Metric) - emit!(crate::internal_events::SinkRequestBuildError { - error: "Unsupported event type for OTLP gRPC sink (native Vector Metric events are not supported; use OTLP-decoded metrics)", - }); - futures::future::ready(None) + // Event does not contain OTLP structure (missing resourceLogs, + // resourceMetrics, or resourceSpans field). This sink only accepts + // OTLP-structured events; plain log events from non-OTLP sources + // must be encoded with an OTLP codec before reaching this sink. + drop_event!( + "event is not OTLP-encoded (missing resourceLogs, resourceMetrics, \ + or resourceSpans field); this sink only accepts OTLP-structured events" + ); } Err(e) => { - emit!(crate::internal_events::SinkRequestBuildError { error: e }); - futures::future::ready(None) + drop_event!(e); } } }) @@ -818,34 +855,17 @@ where self.batch_settings.timeout, |_| self.batch_settings.as_byte_size_config(), ) - .flat_map(|(key, items)| { - // Re-parse the URI from the partition key (always succeeds: it was validated - // in filter_map before being stored in the key). - let uri: Uri = match key.uri.parse::() { - Ok(u) => u, - Err(e) => { - error!("Failed to re-parse gRPC batch partition URI: {e}"); - return futures::stream::iter(Vec::new()); - } + .flat_map(|(_, mut items)| { + // All items in a partition share the same URI and dynamic headers by construction; + // take them from the first item rather than re-parsing from the partition key. + let (uri, dynamic_headers) = match items.first() { + Some(first) => (first.uri.clone(), first.dynamic_headers.clone()), + None => return futures::stream::iter(Vec::new()), }; - let dynamic_headers: Vec<( - tonic::metadata::AsciiMetadataKey, - tonic::metadata::AsciiMetadataValue, - )> = key - .headers - .iter() - .filter_map(|(k, v)| { - Some(( - tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()).ok()?, - tonic::metadata::AsciiMetadataValue::try_from(v.as_str()).ok()?, - )) - }) - .collect(); - // Reduce the partitioned items into per-signal accumulators. let mut batch = OtlpBatch::default(); - for item in items { + for item in items.drain(..) { batch.push(item); } @@ -863,7 +883,7 @@ where data.json_byte_size, ); requests.push(OtlpGrpcRequest { - signal: OtlpSignalRequest::$variant(data.request), + signal: OtlpSignal::$variant(data.request), uri: uri.clone(), dynamic_headers: dynamic_headers.clone(), finalizers: data.finalizers, diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs index aa29f5d0d5457..45993ff0acf28 100644 --- a/src/sinks/opentelemetry/integration_tests.rs +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -80,6 +80,8 @@ async fn delivers_logs_via_grpc() { let (sink, _healthcheck) = config.build(SinkContext::default()).await.unwrap(); let events = vec![otlp_log_event()]; + // The gRPC sink emits EndpointBytesSent with the same "endpoint" and "protocol" tags as + // HTTP sinks, so HTTP_SINK_TAGS is the correct compliance set here. run_and_assert_sink_compliance(sink, stream::iter(events), &HTTP_SINK_TAGS).await; } @@ -88,9 +90,11 @@ async fn delivers_logs_via_grpc_template_uri() { let host = sink_grpc_address(); wait_for_tcp(host.clone()).await; - let config: GrpcSinkConfig = toml::from_str(r#" + let config: GrpcSinkConfig = toml::from_str( + r#" uri = "http://{{ host }}:4317" - "#) + "#, + ) .unwrap(); let (sink, _healthcheck) = config.build(SinkContext::default()).await.unwrap(); diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 8f91b17800a87..379a3f30ea628 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -90,7 +90,9 @@ pub struct OpenTelemetryConfig { /// - `http://localhost:4317` (gRPC) #[configurable(metadata(docs::examples = "http://localhost:5318/v1/logs"))] #[configurable(metadata(docs::examples = "http://localhost:4317"))] - #[configurable(metadata(docs::warnings = "When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations."))] + #[configurable(metadata( + docs::warnings = "When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations." + ))] pub uri: Template, #[configurable(derived)] diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index bb02c12f802da..415ab9b8fddc4 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1116,10 +1116,11 @@ generated: components: sinks: opentelemetry: configuration: { description: """ The URI to send requests to. - Must include a scheme (`http://` or `https://`) and a port. + Supports template syntax (e.g. `http://{{ host }}:4317`). Must include a scheme + (`http://` or `https://`) and a port. - For the HTTP transport, template syntax is supported (e.g. `http://{{ host }}:4318/v1/logs`). - The gRPC transport requires a literal URI; template syntax is not supported. + For the gRPC transport, the template is rendered once per batch using the first event + in the batch. # Examples @@ -1131,5 +1132,6 @@ generated: components: sinks: opentelemetry: configuration: { examples: ["http://localhost:5318/v1/logs", "http://localhost:4317"] syntax: "template" } + warnings: ["When using template syntax, the rendered URI is taken from event data. Only use dynamic URIs with trusted event sources to avoid directing Vector to unintended internal network destinations."] } } From 7e3ac8f5660b07d02fa099b681c9da362315f002 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 16:23:28 -0400 Subject: [PATCH 29/68] remove todos --- ...1-pending-p1-ssrf-dynamic-uri-rendering.md | 57 ----------------- ...pending-p1-tls-mode-fixed-at-build-time.md | 61 ------------------- 2 files changed, 118 deletions(-) delete mode 100644 todos/001-pending-p1-ssrf-dynamic-uri-rendering.md delete mode 100644 todos/002-pending-p1-tls-mode-fixed-at-build-time.md diff --git a/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md b/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md deleted file mode 100644 index 987e13d17d6a9..0000000000000 --- a/todos/001-pending-p1-ssrf-dynamic-uri-rendering.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -name: SSRF via unvalidated dynamic URI rendering -description: Dynamic gRPC URI rendered from untrusted event fields with no allowlist or scheme restriction, enabling SSRF -type: finding -status: resolved -priority: p1 -issue_id: "001" -tags: [code-review, security, opentelemetry, grpc] ---- - -## Problem Statement - -When `uri` is a template (e.g. `http://{{ host }}:4317`), the rendered value is taken verbatim from the event field and only validated as a syntactically valid `Uri`. There is no allowlist, no scheme restriction, and no hostname restriction applied to the rendered result. - -An attacker who controls event data (e.g. via a log ingestion pipeline that accepts external input) can inject any value into the `host` field, directing Vector to make gRPC TCP connections to arbitrary internal network hosts including cloud metadata endpoints (`169.254.169.254`), internal Kubernetes services, etc. - -The same primitive applies to dynamic header values — while constrained to ASCII, they are not restricted beyond that type check. - -## Findings - -- **File**: `src/sinks/opentelemetry/grpc.rs` lines 711-736 -- **Mechanism**: `uri_template.render_string(&event)` → `rendered.parse::()` → used directly for connection -- **Documented use case**: per-tenant routing with `{{ host }}` — explicitly designed for scenarios where event data controls egress destination -- **No allowlist or scheme restriction in the URI validation path** - -## Proposed Solutions - -### Option A: URI allowlist config key -Add `allowed_uri_prefixes: Vec` to `GrpcSinkConfig`. Any rendered URI that doesn't match a prefix is dropped with `SinkRequestBuildError`. Fail closed. -- **Pros**: Flexible, operator-controlled, explicit -- **Cons**: Breaking addition to config, operators must configure it to use dynamic URIs -- **Effort**: Medium -- **Risk**: Low (additive) - -### Option B: Restrict dynamic URI to scheme + host changes only -Parse the rendered URI and validate scheme is `http` or `https` only, and optionally restrict port range. Reject file://, ftp://, etc. -- **Pros**: Simpler than allowlist, catches obvious SSRF vectors -- **Cons**: Doesn't prevent SSRF to internal network; `http://169.254.169.254:80` still valid -- **Effort**: Small -- **Risk**: Low - -### Option C: Document as operator responsibility, add security warning to config docs -Add a `#[configurable(metadata(docs::warnings = "..."))]` annotation noting that dynamic URI templates should only be used with trusted event data. -- **Pros**: No code change -- **Cons**: Does not prevent the vulnerability; just warns -- **Effort**: Small -- **Risk**: None - -## Acceptance Criteria - -- [x] Dynamic URI rendering is either restricted (allowlist) or clearly documented as requiring trusted input -- [x] At minimum, scheme is validated to be `http` or `https` only for rendered URIs -- [x] Any rendered URI that fails validation emits `SinkRequestBuildError` and drops the event - -## Work Log - -- 2026-03-26: Identified by security-sentinel review agent diff --git a/todos/002-pending-p1-tls-mode-fixed-at-build-time.md b/todos/002-pending-p1-tls-mode-fixed-at-build-time.md deleted file mode 100644 index 3598f1ca63149..0000000000000 --- a/todos/002-pending-p1-tls-mode-fixed-at-build-time.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: TLS mode fixed at build time silently sends plaintext for some dynamic URI templates -description: use_https flag computed once at build time; templates like {{ scheme }}://{{ host }} may silently connect over plaintext -type: finding -status: resolved -priority: p1 -issue_id: "002" -tags: [code-review, security, tls, opentelemetry, grpc] ---- - -## Problem Statement - -The `use_https` flag is computed once at sink build time: - -```rust -let use_https = self.tls.is_some() - || static_uri.as_ref().is_some_and(|u| u.scheme_str() == Some("https")) - || self.uri.get_ref().starts_with("https://"); -``` - -The third check (`starts_with("https://")`) only matches when the literal template string begins with `https://`. A template like `{{ scheme }}://{{ host }}:4317` or even `http://{{ host }}:4317` where an event renders to `https://...` will have `use_https = false`. The underlying Hyper client is then built with `MaybeTlsSettings::Raw(())` — a plaintext connector — regardless of the rendered URI scheme. - -This means operators using fully-dynamic URI templates who believe they have TLS will silently send telemetry in plaintext. - -## Findings - -- **File**: `src/sinks/opentelemetry/grpc.rs` lines 155-169 -- **Risk**: Silent plaintext egress for PII/confidential telemetry -- **Affected configurations**: Any dynamic URI template where the scheme is not a literal static prefix - -## Proposed Solutions - -### Option A: Validate rendered URI scheme matches build-time TLS mode -At event render time, check that the rendered URI's scheme matches `use_https`. If they conflict, drop the event with a `SinkRequestBuildError` and a clear message. -- **Pros**: Fails loudly, no silent plaintext -- **Cons**: Drops events when config is ambiguous; operator must configure `tls:` block or use static scheme -- **Effort**: Small -- **Risk**: Low - -### Option B: Require scheme to be a static literal in dynamic URI templates -At config parse time, validate that the scheme portion of any template URI is not itself a template expression. Reject configs like `{{ scheme }}://{{ host }}`. -- **Pros**: Eliminates the ambiguity at startup -- **Cons**: Restricts valid template patterns -- **Effort**: Medium (requires template parsing) -- **Risk**: Low - -### Option C: Document the limitation clearly in config schema -Add a docstring note to the `uri` field: "When using a dynamic template that renders to `https://`, you must also configure `tls:` to ensure TLS is used. The sink cannot infer TLS from a fully-dynamic scheme." -- **Pros**: No code change, low effort -- **Cons**: Doesn't prevent the issue, just warns -- **Effort**: Small -- **Risk**: None - -## Acceptance Criteria - -- [x] Operators are not silently sent over plaintext when they configure a dynamic `https://` URI template -- [x] Either the scheme mismatch is detected and rejected at runtime, or config validation prevents it - -## Work Log - -- 2026-03-26: Identified by security-sentinel review agent From 99d2b0717f171cf72371689cdb1209f5472a7fd6 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 16:39:08 -0400 Subject: [PATCH 30/68] docs(opentelemetry sink): warn that gRPC only supports none and gzip compression --- src/sinks/opentelemetry/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 379a3f30ea628..c0f128f2b8717 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -96,6 +96,7 @@ pub struct OpenTelemetryConfig { pub uri: Template, #[configurable(derived)] + #[configurable(metadata(docs::warnings = "The `grpc` protocol only supports `none` and `gzip`. Specifying any other algorithm causes Vector to fail at startup."))] #[serde(default)] pub compression: Compression, @@ -160,8 +161,7 @@ impl SinkConfig for OpenTelemetryConfig { Compression::Gzip(_) => GrpcCompression::Gzip, other => return Err(format!( "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" - ) - .into()), + ).into()), }; let config = GrpcSinkConfig { uri: self.uri.clone(), From dc5e9e3701791d6ce6ee3eebf4948c41b282c39c Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 16:42:19 -0400 Subject: [PATCH 31/68] fix(opentelemetry sink): normalize use_https prefix check to lowercase --- src/sinks/opentelemetry/grpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index e7dbaf41c771f..a8d94fac36dc9 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -152,7 +152,7 @@ impl GrpcSinkConfig { || static_uri .as_ref() .is_some_and(|u| u.scheme_str() == Some("https")) - || self.uri.get_ref().starts_with("https://"); + || self.uri.get_ref().to_ascii_lowercase().starts_with("https://"); let tls = if use_https { MaybeTlsSettings::tls_client(self.tls.as_ref())? From f88fcc40a5c33926b59b620cd5ad5d4175244c46 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 16:44:26 -0400 Subject: [PATCH 32/68] fix(opentelemetry sink): warn instead of debug when skipping gRPC healthcheck for dynamic URI --- src/sinks/opentelemetry/grpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index a8d94fac36dc9..4eae50da6cb9d 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -254,7 +254,10 @@ async fn grpc_healthcheck( } let Some(uri) = uri else { - debug!("Skipping gRPC healthcheck: URI is a dynamic template resolved at runtime."); + warn!( + "Skipping gRPC healthcheck: `uri` is a dynamic template and cannot be validated \ + at startup. To enable healthchecking, use a static URI." + ); return Ok(()); }; From fbf1f151730ec7af2498761e1807fb0ab3bf6b9e Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 16:48:52 -0400 Subject: [PATCH 33/68] perf(opentelemetry sink): wrap static gRPC headers in Arc to avoid per-request Vec clone --- src/sinks/opentelemetry/grpc.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 4eae50da6cb9d..c879a62d069ce 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -397,10 +397,10 @@ pub struct OtlpGrpcService { clients: std::sync::Arc>>, hyper_client: hyper::Client>, BoxBody>, compression: bool, - headers: Vec<( + headers: std::sync::Arc, + )>>, } impl OtlpGrpcService { @@ -416,7 +416,7 @@ impl OtlpGrpcService { clients: std::sync::Arc::new(std::sync::Mutex::new(None)), hyper_client, compression, - headers, + headers: std::sync::Arc::new(headers), } } @@ -469,7 +469,7 @@ impl Service for OtlpGrpcService { let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(req.uri.clone()); // Rebuild clients if the URI changed; clone them out before any `.await`. let (mut logs_client, mut metrics_client, mut traces_client) = self.clients_for(&req.uri); - let static_headers = self.headers.clone(); + let static_headers = std::sync::Arc::clone(&self.headers); let metadata = std::mem::take(req.metadata_mut()); let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); @@ -478,7 +478,7 @@ impl Service for OtlpGrpcService { ($client:expr, $payload:expr) => {{ let len = $payload.encoded_len(); let mut grpc_req = tonic::Request::new($payload); - for (key, value) in &static_headers { + for (key, value) in static_headers.iter() { grpc_req.metadata_mut().insert(key.clone(), value.clone()); } for (key, value) in &req.dynamic_headers { From 050cf32e40ef2820ede5e1dd58cdc3505f089752 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 17:33:27 -0400 Subject: [PATCH 34/68] fix(opentelemetry sink): handle OTLP spans arriving as Log events in gRPC sink and serializer --- lib/codecs/src/encoding/format/otlp.rs | 7 ++++++- src/sinks/opentelemetry/grpc.rs | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/codecs/src/encoding/format/otlp.rs b/lib/codecs/src/encoding/format/otlp.rs index fd0ffffe47b04..bb06c5d1cf4ae 100644 --- a/lib/codecs/src/encoding/format/otlp.rs +++ b/lib/codecs/src/encoding/format/otlp.rs @@ -107,9 +107,14 @@ impl Encoder for OtlpSerializer { } else if log.contains(RESOURCE_METRICS_JSON_FIELD) { // Currently the OTLP metrics are Vector logs (not metrics). self.metrics_descriptor.encode(event, buffer) + } else if log.contains(RESOURCE_SPANS_JSON_FIELD) { + // OTLP spans can arrive as Log events when the source is configured + // without use_otlp_decoding.traces = true. + self.traces_descriptor.encode(event, buffer) } else { Err(format!( - "Log event does not contain OTLP top-level fields ({RESOURCE_LOGS_JSON_FIELD} or {RESOURCE_METRICS_JSON_FIELD})", + "Log event does not contain OTLP top-level fields \ + ({RESOURCE_LOGS_JSON_FIELD}, {RESOURCE_METRICS_JSON_FIELD}, or {RESOURCE_SPANS_JSON_FIELD})", ) .into()) } diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index c879a62d069ce..f2277674ef1dc 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -972,6 +972,10 @@ fn detect_signal_type(event: &Event) -> Option { Some(SignalType::Logs) } else if log.contains(RESOURCE_METRICS_JSON_FIELD) { Some(SignalType::Metrics) + } else if log.contains(RESOURCE_SPANS_JSON_FIELD) { + // OTLP spans can arrive as Log events when the source does not use + // use_otlp_decoding.traces = true. + Some(SignalType::Traces) } else { None } From 68e86f1fdcd8e1feade4a29b0949fbe1a2e88833 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 17:35:20 -0400 Subject: [PATCH 35/68] refactor(opentelemetry sink): flatten URI validation chain in gRPC sink run_inner --- src/sinks/opentelemetry/grpc.rs | 71 +++++++++++++++------------------ 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index f2277674ef1dc..8a5afe7f956e2 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -749,44 +749,39 @@ where }}; } - let uri = match uri_template.render_string(&event) { - Ok(rendered) => match rendered.parse::() { - Ok(parsed) => match with_default_scheme(parsed, use_https) { - Ok(u) => { - match u.scheme_str() { - Some("https") if !use_https => { - // The Hyper client was built without TLS (use_https=false), - // so it cannot complete a TLS handshake. Sending data to an - // https:// endpoint over a plaintext connector would either - // fail or silently transmit unencrypted. Drop the event and - // surface a clear error so the operator can add `tls:` or - // use a static `https://` scheme prefix. - drop_event!( - "rendered gRPC URI uses \"https\" but the sink \ - has no TLS connector; add a `tls:` block or use \ - a static \"https://\" URI prefix so TLS is \ - enabled at startup" - ); - } - Some("http") | Some("https") => u, - other => { - drop_event!(format!( - "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", - other.unwrap_or("") - )); - } - } - } - Err(e) => { - drop_event!(format!("invalid gRPC URI after rendering template: {e}")); - } - }, - Err(e) => { - drop_event!(format!("failed to parse rendered gRPC URI: {e}")); - } - }, - Err(e) => { - drop_event!(format!("failed to render gRPC URI template: {e}")); + let rendered = match uri_template.render_string(&event) { + Ok(s) => s, + Err(e) => drop_event!(format!("failed to render gRPC URI template: {e}")), + }; + let parsed = match rendered.parse::() { + Ok(u) => u, + Err(e) => drop_event!(format!("failed to parse rendered gRPC URI: {e}")), + }; + let u = match with_default_scheme(parsed, use_https) { + Ok(u) => u, + Err(e) => drop_event!(format!("invalid gRPC URI after rendering template: {e}")), + }; + let uri = match u.scheme_str() { + Some("https") if !use_https => { + // The Hyper client was built without TLS (use_https=false), + // so it cannot complete a TLS handshake. Sending data to an + // https:// endpoint over a plaintext connector would either + // fail or silently transmit unencrypted. Drop the event and + // surface a clear error so the operator can add `tls:` or + // use a static `https://` scheme prefix. + drop_event!( + "rendered gRPC URI uses \"https\" but the sink \ + has no TLS connector; add a `tls:` block or use \ + a static \"https://\" URI prefix so TLS is \ + enabled at startup" + ); + } + Some("http") | Some("https") => u, + other => { + drop_event!(format!( + "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", + other.unwrap_or("") + )); } }; From 7a91179e031c8e28f7abd78bf3cc07766188395f Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 17:49:11 -0400 Subject: [PATCH 36/68] refactor(sinks): extract shared HyperGrpcService to sinks/util/grpc --- src/sinks/opentelemetry/grpc.rs | 44 ++------------------------ src/sinks/util/grpc.rs | 56 +++++++++++++++++++++++++++++++++ src/sinks/util/mod.rs | 1 + src/sinks/vector/service.rs | 31 +----------------- 4 files changed, 60 insertions(+), 72 deletions(-) create mode 100644 src/sinks/util/grpc.rs diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 8a5afe7f956e2..57f0c4593e741 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -42,6 +42,7 @@ use vector_lib::{ use crate::{ config::{AcknowledgementsConfig, DataType, Input, SinkContext, SinkHealthcheckOptions}, + sinks::util::grpc::HyperGrpcService, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, internal_events::EndpointBytesSent, @@ -511,48 +512,7 @@ impl Service for OtlpGrpcService { } } -// ── HyperSvc (same as in sinks/vector/service.rs) ──────────────────────────── - -#[derive(Clone)] -pub struct HyperSvc { - uri: Uri, - client: hyper::Client>, BoxBody>, -} - -impl Service> for HyperSvc { - type Response = hyper::Response; - type Error = hyper::Error; - type Future = BoxFuture<'static, Result>; - - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, mut req: hyper::Request) -> Self::Future { - // SAFETY: `self.uri` is always produced by `with_default_scheme`, which guarantees - // a scheme and a path_and_query. Tonic always sets a path on the request URI. - let uri = Uri::builder() - .scheme(self.uri.scheme().expect("uri always has a scheme").clone()) - .authority( - self.uri - .authority() - .expect("uri always has an authority") - .clone(), - ) - .path_and_query( - req.uri() - .path_and_query() - .expect("tonic request always has a path") - .clone(), - ) - .build() - .expect("uri components are always valid"); - - *req.uri_mut() = uri; - - Box::pin(self.client.request(req)) - } -} +type HyperSvc = HyperGrpcService; // ── Sink ───────────────────────────────────────────────────────────────────── diff --git a/src/sinks/util/grpc.rs b/src/sinks/util/grpc.rs new file mode 100644 index 0000000000000..61d32b2fd6871 --- /dev/null +++ b/src/sinks/util/grpc.rs @@ -0,0 +1,56 @@ +use std::task::{Context, Poll}; + +use futures::future::BoxFuture; +use http::Uri; +use hyper::client::HttpConnector; +use hyper_openssl::HttpsConnector; +use hyper_proxy::ProxyConnector; +use tonic::body::BoxBody; +use tower::Service; + +/// A Tower [`Service`] that routes gRPC requests through a Hyper HTTP/2 client, +/// substituting the scheme and authority from a fixed base URI while preserving +/// the path/query set by tonic. +/// +/// Used by gRPC sinks that need to send requests to a specific endpoint using a +/// shared Hyper client. +#[derive(Clone, Debug)] +pub struct HyperGrpcService { + pub uri: Uri, + pub client: hyper::Client>, BoxBody>, +} + +impl Service> for HyperGrpcService { + type Response = hyper::Response; + type Error = hyper::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, mut req: hyper::Request) -> Self::Future { + // SAFETY: `self.uri` is always produced by `with_default_scheme` or equivalent, + // which guarantees a scheme and authority. Tonic always sets a path on the request URI. + let uri = Uri::builder() + .scheme(self.uri.scheme().expect("uri always has a scheme").clone()) + .authority( + self.uri + .authority() + .expect("uri always has an authority") + .clone(), + ) + .path_and_query( + req.uri() + .path_and_query() + .expect("tonic request always has a path") + .clone(), + ) + .build() + .expect("uri components are always valid"); + + *req.uri_mut() = uri; + + Box::pin(self.client.request(req)) + } +} diff --git a/src/sinks/util/mod.rs b/src/sinks/util/mod.rs index 5d766a9b30780..439e56606827f 100644 --- a/src/sinks/util/mod.rs +++ b/src/sinks/util/mod.rs @@ -1,5 +1,6 @@ pub mod adaptive_concurrency; pub mod auth; +pub mod grpc; // https://github.com/mcarton/rust-derivative/issues/112 #[allow(clippy::non_canonical_clone_impl)] pub mod batch; diff --git a/src/sinks/vector/service.rs b/src/sinks/vector/service.rs index 471b3a98ed6b1..33c310fcbf321 100644 --- a/src/sinks/vector/service.rs +++ b/src/sinks/vector/service.rs @@ -132,33 +132,4 @@ impl Service for VectorService { } } -#[derive(Clone, Debug)] -pub struct HyperSvc { - uri: Uri, - client: hyper::Client>, BoxBody>, -} - -impl Service> for HyperSvc { - type Response = hyper::Response; - type Error = hyper::Error; - type Future = BoxFuture<'static, Result>; - - // Emission of an internal event in case of errors is handled upstream by the caller. - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - // Emission of internal events for errors and dropped events is handled upstream by the caller. - fn call(&mut self, mut req: hyper::Request) -> Self::Future { - let uri = Uri::builder() - .scheme(self.uri.scheme().unwrap().clone()) - .authority(self.uri.authority().unwrap().clone()) - .path_and_query(req.uri().path_and_query().unwrap().clone()) - .build() - .unwrap(); - - *req.uri_mut() = uri; - - Box::pin(self.client.request(req)) - } -} +pub use crate::sinks::util::grpc::HyperGrpcService as HyperSvc; From ddb9cab84006212bab2681c04f05cbf93bfa064a Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 18:02:37 -0400 Subject: [PATCH 37/68] docs(opentelemetry sink): remove incorrect 'Defaults to http' on required protocol field --- src/sinks/opentelemetry/mod.rs | 2 +- .../cue/reference/components/sinks/generated/opentelemetry.cue | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index c0f128f2b8717..7b78fbfa4f6ef 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -71,7 +71,7 @@ pub enum OtlpProtocol { #[configurable_component(sink("opentelemetry", "Deliver OTLP data over HTTP or gRPC."))] #[derive(Clone, Debug)] pub struct OpenTelemetryConfig { - /// The transport protocol to use. Defaults to `http`. + /// The transport protocol to use. #[configurable(derived)] #[serde(flatten)] pub protocol: OtlpProtocol, diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 415ab9b8fddc4..9a3a207767e14 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -277,6 +277,7 @@ generated: components: sinks: opentelemetry: configuration: { """ } } + warnings: ["The `grpc` protocol only supports `none` and `gzip`. Specifying any other algorithm causes Vector to fail at startup."] } encoding: { description: """ From 26d490f193ad5b46fe27d2a84d2edfd2ab42a8f5 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 18:07:50 -0400 Subject: [PATCH 38/68] fix(opentelemetry sink): reject invalid static gRPC metadata headers at build time --- src/sinks/opentelemetry/grpc.rs | 43 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 57f0c4593e741..7c23b4adeb51d 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -169,31 +169,30 @@ impl GrpcSinkConfig { // resolve correctly at export time. let (static_header_strings, dynamic_header_templates_raw) = self.request.split_headers(); - let parse_key = |k: &str| { - tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) - .map_err(|e| warn!("Skipping invalid gRPC metadata key {k:?}: {e}")) - .ok() - }; - - let static_grpc_headers: Vec<( + // Static headers are validated at build time and any invalid key or value is a hard + // error. Silently dropping them would cause every request to be sent without the + // intended metadata (e.g. auth headers), which is worse than failing fast. + let mut static_grpc_headers: Vec<( tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue, - )> = static_header_strings - .iter() - .filter_map(|(k, v)| { - let key = parse_key(k)?; - let value = tonic::metadata::AsciiMetadataValue::try_from(v.as_str()) - .map_err(|e| warn!("Skipping invalid gRPC metadata value for {k:?}: {e}")) - .ok()?; - Some((key, value)) - }) - .collect(); + )> = Vec::with_capacity(static_header_strings.len()); + for (k, v) in &static_header_strings { + let key = tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) + .map_err(|e| format!("invalid gRPC metadata key {k:?}: {e}"))?; + let value = tonic::metadata::AsciiMetadataValue::try_from(v.as_str()) + .map_err(|e| format!("invalid gRPC metadata value for key {k:?}: {e}"))?; + static_grpc_headers.push((key, value)); + } - let dynamic_grpc_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)> = - dynamic_header_templates_raw - .into_iter() - .filter_map(|(k, t)| Some((parse_key(&k)?, t))) - .collect(); + // Dynamic header key names are known at build time and validated eagerly. + // Values are templated and validated per-event at runtime. + let mut dynamic_grpc_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)> = + Vec::with_capacity(dynamic_header_templates_raw.len()); + for (k, t) in dynamic_header_templates_raw { + let key = tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) + .map_err(|e| format!("invalid gRPC metadata key {k:?}: {e}"))?; + dynamic_grpc_header_templates.push((key, t)); + } let client = new_grpc_client(&tls, cx.proxy())?; let healthcheck = Box::pin(grpc_healthcheck( From 33d815dcebaeed2fd40dd0c417ca950aa4862b7b Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 26 Mar 2026 18:09:15 -0400 Subject: [PATCH 39/68] fix(opentelemetry sink): migrate stale config fixtures from nested protocol.type to flat format --- .../data/vector_otlp.yaml | 9 ++++----- .../en/highlights/2025-09-23-otlp-support.md | 20 +++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/e2e/opentelemetry-metrics/data/vector_otlp.yaml b/tests/e2e/opentelemetry-metrics/data/vector_otlp.yaml index 0dec0745c3a13..2900326e7d346 100644 --- a/tests/e2e/opentelemetry-metrics/data/vector_otlp.yaml +++ b/tests/e2e/opentelemetry-metrics/data/vector_otlp.yaml @@ -19,11 +19,10 @@ sinks: inputs: - source0.metrics type: opentelemetry - protocol: - type: http - uri: http://otel-collector-sink:5318/v1/metrics - encoding: - codec: otlp + protocol: http + uri: http://otel-collector-sink:5318/v1/metrics + encoding: + codec: otlp otel_file_sink: type: file diff --git a/website/content/en/highlights/2025-09-23-otlp-support.md b/website/content/en/highlights/2025-09-23-otlp-support.md index fa3a1add99544..4f217490efa92 100644 --- a/website/content/en/highlights/2025-09-23-otlp-support.md +++ b/website/content/en/highlights/2025-09-23-otlp-support.md @@ -39,12 +39,11 @@ sinks: inputs: - source0.logs type: opentelemetry - protocol: - type: http - uri: http://otel-collector-sink:5318/v1/logs - method: post - encoding: - codec: otlp + protocol: http + uri: http://otel-collector-sink:5318/v1/logs + method: post + encoding: + codec: otlp ``` The above configuration will only work with Vector versions >= `0.51`. @@ -58,11 +57,10 @@ otel_sink: inputs: - otel.logs type: opentelemetry - protocol: - type: http - uri: http://localhost:5318/v1/logs - method: post - encoding: + protocol: http + uri: http://localhost:5318/v1/logs + method: post + encoding: codec: protobuf protobuf: desc_file: path/to/opentelemetry-proto.desc From f9a7079dafcd50c8440d09b21c4c08b7a322195d Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 14:12:55 -0400 Subject: [PATCH 40/68] fix(opentelemetry sink): tighten TLS heuristic, warn on OTLP partial-success, restrict GrpcSinkConfig visibility, document gRPC request field --- src/sinks/opentelemetry/grpc.rs | 54 ++++++++++++++++++++------------- src/sinks/opentelemetry/mod.rs | 7 ++++- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 7c23b4adeb51d..e9884e6c945db 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -41,7 +41,7 @@ use vector_lib::{ }; use crate::{ - config::{AcknowledgementsConfig, DataType, Input, SinkContext, SinkHealthcheckOptions}, + config::{AcknowledgementsConfig, SinkContext, SinkHealthcheckOptions}, sinks::util::grpc::HyperGrpcService, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, @@ -88,7 +88,7 @@ pub(super) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { /// Configuration for the OpenTelemetry sink's gRPC transport. #[configurable_component] #[derive(Clone, Debug)] -pub struct GrpcSinkConfig { +pub(super) struct GrpcSinkConfig { /// The URI to send gRPC requests to. /// /// The URI _must_ include a port. If the scheme is omitted, `http` is used unless @@ -130,7 +130,7 @@ pub struct GrpcSinkConfig { } impl GrpcSinkConfig { - pub async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + pub(super) async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { // For static URIs, parse at build time for the healthcheck. // Dynamic URIs are rendered per-event during sink execution. let static_uri = if self.uri.is_dynamic() { @@ -146,14 +146,20 @@ impl GrpcSinkConfig { }; // For dynamic templates like `https://{{ host }}:4317` the static_uri is None, so - // we also check whether the literal prefix of the template string is "https://". - // This covers the common case where the scheme is a fixed literal even though the - // host/port are templated. + // we also check whether the static literal prefix of the template string starts with + // "https://". Only the part before the first `{{` is inspected so that a template + // like `{{ scheme }}://host:4317` does not falsely trigger TLS — in that case the + // scheme is not known at build time and events will be dropped at runtime if a + // rendered `https://` URI is encountered without a TLS connector. let use_https = self.tls.is_some() || static_uri .as_ref() .is_some_and(|u| u.scheme_str() == Some("https")) - || self.uri.get_ref().to_ascii_lowercase().starts_with("https://"); + || { + let raw = self.uri.get_ref().to_ascii_lowercase(); + let static_prefix_end = raw.find("{{").unwrap_or(raw.len()); + raw[..static_prefix_end].contains("https://") + }; let tls = if use_https { MaybeTlsSettings::tls_client(self.tls.as_ref())? @@ -221,15 +227,6 @@ impl GrpcSinkConfig { Ok((VectorSink::from_event_streamsink(sink), healthcheck)) } - pub fn input(&self) -> Input { - // Native Vector Metric events are not supported; OTLP-encoded metrics arrive as Log - // events with a `resourceMetrics` field and are handled correctly. - Input::new(DataType::Log | DataType::Trace) - } - - pub const fn acknowledgements(&self) -> &AcknowledgementsConfig { - &self.acknowledgements - } } fn new_grpc_client( @@ -484,18 +481,33 @@ impl Service for OtlpGrpcService { for (key, value) in &req.dynamic_headers { grpc_req.metadata_mut().insert(key.clone(), value.clone()); } - $client + let response = $client .export(grpc_req) .map_err(|source| OtlpGrpcError::Request { source }) .await?; - len + (len, response.into_inner()) }}; } let byte_size = match req.signal { - OtlpSignal::Logs(r) => export!(logs_client, r), - OtlpSignal::Metrics(r) => export!(metrics_client, r), - OtlpSignal::Traces(r) => export!(traces_client, r), + OtlpSignal::Logs(r) => { + let (len, resp) = export!(logs_client, r); + let rejected = resp.partial_success.map(|ps| ps.rejected_log_records).unwrap_or(0); + if rejected > 0 { warn!(rejected, "OTLP collector rejected log records"); } + len + } + OtlpSignal::Metrics(r) => { + let (len, resp) = export!(metrics_client, r); + let rejected = resp.partial_success.map(|ps| ps.rejected_data_points).unwrap_or(0); + if rejected > 0 { warn!(rejected, "OTLP collector rejected metric data points"); } + len + } + OtlpSignal::Traces(r) => { + let (len, resp) = export!(traces_client, r); + let rejected = resp.partial_success.map(|ps| ps.rejected_spans).unwrap_or(0); + if rejected > 0 { warn!(rejected, "OTLP collector rejected trace spans"); } + len + } }; emit!(EndpointBytesSent { diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 7b78fbfa4f6ef..3f450e95fff67 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -23,7 +23,8 @@ use crate::{ tls::TlsConfig, }; -pub use grpc::{GrpcCompression, GrpcSinkConfig}; +pub use grpc::GrpcCompression; +use grpc::GrpcSinkConfig; /// Transport protocol for the OpenTelemetry sink. #[configurable_component] @@ -101,6 +102,10 @@ pub struct OpenTelemetryConfig { pub compression: Compression, #[configurable(derived)] + #[configurable(metadata(docs::description = "Outbound request settings for retry, concurrency, timeout, and headers. \ + For the `grpc` protocol, `request.headers` entries are forwarded as gRPC metadata — use them \ + for authentication (e.g. `authorization: \"Bearer \"`) since the HTTP-only `auth` field \ + is not available for gRPC."))] #[serde(default)] pub request: RequestConfig, From e8260a59845b6e255ab46f6cd3320b0ad34dc215 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 14:37:55 -0400 Subject: [PATCH 41/68] =?UTF-8?q?perf(opentelemetry=20sink):=20remove=20Ar?= =?UTF-8?q?c=20from=20gRPC=20client=20cache=20=E2=80=94=20use=20per?= =?UTF-8?q?-clone=20Option?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sinks/opentelemetry/grpc.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index e9884e6c945db..02a5147143334 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -381,6 +381,7 @@ impl DriverResponse for OtlpGrpcResponse { // ── Service ────────────────────────────────────────────────────────────────── +#[derive(Clone)] struct CachedClients { uri: Uri, logs: LogsServiceClient, @@ -391,7 +392,8 @@ struct CachedClients { #[derive(Clone)] pub struct OtlpGrpcService { /// Tonic clients for the most-recently-seen URI; rebuilt when the rendered URI changes. - clients: std::sync::Arc>>, + /// Each Tower concurrency-slot clone owns its cache independently — no shared mutex needed. + clients: Option, hyper_client: hyper::Client>, BoxBody>, compression: bool, headers: std::sync::Arc, ) -> Self { Self { - clients: std::sync::Arc::new(std::sync::Mutex::new(None)), + clients: None, hyper_client, compression, headers: std::sync::Arc::new(headers), @@ -418,17 +420,15 @@ impl OtlpGrpcService { } /// Returns cloned tonic clients for `uri`, rebuilding them if the URI changed. - /// The mutex is held only during the synchronous rebuild, never across any `.await`. fn clients_for( - &self, + &mut self, uri: &Uri, ) -> ( LogsServiceClient, MetricsServiceClient, TraceServiceClient, ) { - let mut guard = self.clients.lock().expect("client lock poisoned"); - if guard.as_ref().is_none_or(|c| &c.uri != uri) { + if self.clients.as_ref().is_none_or(|c| &c.uri != uri) { let svc = HyperSvc { uri: uri.clone(), client: self.hyper_client.clone(), @@ -441,14 +441,14 @@ impl OtlpGrpcService { metrics = metrics.send_compressed(tonic::codec::CompressionEncoding::Gzip); traces = traces.send_compressed(tonic::codec::CompressionEncoding::Gzip); } - *guard = Some(CachedClients { + self.clients = Some(CachedClients { uri: uri.clone(), logs, metrics, traces, }); } - let c = guard.as_ref().expect("just populated"); + let c = self.clients.as_ref().expect("just populated"); (c.logs.clone(), c.metrics.clone(), c.traces.clone()) } } From dc847b323eaf1a2b94025432582fe941a09bfa5d Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 15:30:31 -0400 Subject: [PATCH 42/68] refactor(opentelemetry sink): replace accumulate!/push_signal!/drop_event! macros with named functions --- src/sinks/opentelemetry/grpc.rs | 204 +++++++++++++++----------------- 1 file changed, 98 insertions(+), 106 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 02a5147143334..6b0efd474ac23 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -45,7 +45,7 @@ use crate::{ sinks::util::grpc::HyperGrpcService, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, - internal_events::EndpointBytesSent, + internal_events::{EndpointBytesSent, SinkRequestBuildError}, sinks::{ Healthcheck, VectorSink, util::{ @@ -611,6 +611,32 @@ struct SignalData { json_byte_size: GroupedCountByteSize, } +impl SignalData { + const fn new( + request: R, + finalizers: EventFinalizers, + byte_size: usize, + json_byte_size: GroupedCountByteSize, + ) -> Self { + SignalData { request, finalizers, event_count: 1, byte_size, json_byte_size } + } + + fn merge( + &mut self, + req: R, + finalizers: EventFinalizers, + byte_size: usize, + json_byte_size: GroupedCountByteSize, + merge_req: impl FnOnce(&mut R, R), + ) { + merge_req(&mut self.request, req); + self.finalizers.merge(finalizers); + self.event_count += 1; + self.byte_size += byte_size; + self.json_byte_size += json_byte_size; + } +} + /// Accumulator for a batch of OTLP events, separated by signal type so that /// the resulting requests can be retried independently. #[derive(Default)] @@ -622,60 +648,20 @@ struct OtlpBatch { impl OtlpBatch { fn push(&mut self, item: OtlpEventData) { - let OtlpEventData { - byte_size, - json_byte_size, - finalizers, - signal, - uri: _, - dynamic_headers: _, - } = item; - - macro_rules! accumulate { - ($field:ident, $req:expr, $merge:expr) => { - match &mut self.$field { - Some(existing) => { - $merge(&mut existing.request, $req); - existing.finalizers.merge(finalizers); - existing.event_count += 1; - existing.byte_size += byte_size; - existing.json_byte_size += json_byte_size; - } - slot => { - *slot = Some(SignalData { - request: $req, - finalizers, - event_count: 1, - byte_size, - json_byte_size, - }); - } - } - }; - } - + let OtlpEventData { byte_size, json_byte_size, finalizers, signal, uri: _, dynamic_headers: _ } = item; match signal { - OtlpSignal::Logs(req) => accumulate!( - logs, - req, - |e: &mut ExportLogsServiceRequest, r: ExportLogsServiceRequest| { - e.resource_logs.extend(r.resource_logs) - } - ), - OtlpSignal::Metrics(req) => accumulate!( - metrics, - req, - |e: &mut ExportMetricsServiceRequest, r: ExportMetricsServiceRequest| { - e.resource_metrics.extend(r.resource_metrics) - } - ), - OtlpSignal::Traces(req) => accumulate!( - traces, - req, - |e: &mut ExportTraceServiceRequest, r: ExportTraceServiceRequest| { - e.resource_spans.extend(r.resource_spans) - } - ), + OtlpSignal::Logs(req) => match &mut self.logs { + Some(existing) => existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| e.resource_logs.extend(r.resource_logs)), + slot => *slot = Some(SignalData::new(req, finalizers, byte_size, json_byte_size)), + }, + OtlpSignal::Metrics(req) => match &mut self.metrics { + Some(existing) => existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| e.resource_metrics.extend(r.resource_metrics)), + slot => *slot = Some(SignalData::new(req, finalizers, byte_size, json_byte_size)), + }, + OtlpSignal::Traces(req) => match &mut self.traces { + Some(existing) => existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| e.resource_spans.extend(r.resource_spans)), + slot => *slot = Some(SignalData::new(req, finalizers, byte_size, json_byte_size)), + }, } } } @@ -706,31 +692,17 @@ where input .filter_map(move |mut event| { - macro_rules! drop_event { - ($reason:expr) => {{ - let reason_owned: String = $reason.to_string(); - emit!(crate::internal_events::SinkRequestBuildError { - error: reason_owned.as_str() - }); - emit!(ComponentEventsDropped:: { - count: 1, - reason: reason_owned.as_str(), - }); - return futures::future::ready(None); - }}; - } - let rendered = match uri_template.render_string(&event) { Ok(s) => s, - Err(e) => drop_event!(format!("failed to render gRPC URI template: {e}")), + Err(e) => return reject_event(format!("failed to render gRPC URI template: {e}")), }; let parsed = match rendered.parse::() { Ok(u) => u, - Err(e) => drop_event!(format!("failed to parse rendered gRPC URI: {e}")), + Err(e) => return reject_event(format!("failed to parse rendered gRPC URI: {e}")), }; let u = match with_default_scheme(parsed, use_https) { Ok(u) => u, - Err(e) => drop_event!(format!("invalid gRPC URI after rendering template: {e}")), + Err(e) => return reject_event(format!("invalid gRPC URI after rendering template: {e}")), }; let uri = match u.scheme_str() { Some("https") if !use_https => { @@ -740,16 +712,16 @@ where // fail or silently transmit unencrypted. Drop the event and // surface a clear error so the operator can add `tls:` or // use a static `https://` scheme prefix. - drop_event!( + return reject_event( "rendered gRPC URI uses \"https\" but the sink \ has no TLS connector; add a `tls:` block or use \ a static \"https://\" URI prefix so TLS is \ - enabled at startup" + enabled at startup", ); } Some("http") | Some("https") => u, other => { - drop_event!(format!( + return reject_event(format!( "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", other.unwrap_or("") )); @@ -767,7 +739,7 @@ where match tonic::metadata::AsciiMetadataValue::try_from(rendered.as_str()) { Ok(value) => dynamic_headers.push((key.clone(), value)), Err(e) => { - drop_event!(format!( + return reject_event(format!( "gRPC metadata value for key {:?} is not valid ASCII: {e}", key.as_str() )); @@ -775,7 +747,7 @@ where } } Err(e) => { - drop_event!(format!( + return reject_event(format!( "failed to render gRPC metadata template for key {:?}: {e}", key.as_str() )); @@ -806,14 +778,12 @@ where // resourceMetrics, or resourceSpans field). This sink only accepts // OTLP-structured events; plain log events from non-OTLP sources // must be encoded with an OTLP codec before reaching this sink. - drop_event!( + reject_event( "event is not OTLP-encoded (missing resourceLogs, resourceMetrics, \ - or resourceSpans field); this sink only accepts OTLP-structured events" - ); - } - Err(e) => { - drop_event!(e); + or resourceSpans field); this sink only accepts OTLP-structured events", + ) } + Err(e) => reject_event(e.to_string()), } }) // Partition by (rendered URI, dynamic headers) so that events destined for @@ -840,31 +810,15 @@ where let mut requests = Vec::new(); - macro_rules! push_signal { - ($field:ident, $variant:ident) => { - if let Some(data) = batch.$field { - let byte_size = data.request.encoded_len(); - let bytes_len = - NonZeroUsize::new(byte_size.max(1)).expect("should be non-zero"); - let builder = RequestMetadataBuilder::new( - data.event_count, - data.byte_size, - data.json_byte_size, - ); - requests.push(OtlpGrpcRequest { - signal: OtlpSignal::$variant(data.request), - uri: uri.clone(), - dynamic_headers: dynamic_headers.clone(), - finalizers: data.finalizers, - metadata: builder.with_request_size(bytes_len), - }); - } - }; + if let Some(data) = batch.logs { + requests.push(signal_into_request(data, OtlpSignal::Logs, &uri, dynamic_headers.clone())); + } + if let Some(data) = batch.metrics { + requests.push(signal_into_request(data, OtlpSignal::Metrics, &uri, dynamic_headers.clone())); + } + if let Some(data) = batch.traces { + requests.push(signal_into_request(data, OtlpSignal::Traces, &uri, dynamic_headers.clone())); } - - push_signal!(logs, Logs); - push_signal!(metrics, Metrics); - push_signal!(traces, Traces); futures::stream::iter(requests) }) @@ -957,6 +911,44 @@ fn detect_signal_type(event: &Event) -> Option { } } +/// Drop an event that could not be prepared for export. +/// +/// Emits [`SinkRequestBuildError`] and [`ComponentEventsDropped`], then returns +/// a ready future resolving to `None` so the caller can use `return +/// reject_event(...)` directly inside a `filter_map` closure. +fn reject_event(reason: impl fmt::Display) -> futures::future::Ready> { + let reason = reason.to_string(); + emit!(SinkRequestBuildError { error: &reason }); + emit!(ComponentEventsDropped:: { + count: 1, + reason: &reason, + }); + futures::future::ready(None) +} + +/// Build an [`OtlpGrpcRequest`] from a per-signal accumulator. +fn signal_into_request( + data: SignalData, + make_signal: fn(R) -> OtlpSignal, + uri: &Uri, + dynamic_headers: Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, +) -> OtlpGrpcRequest { + let byte_size = data.request.encoded_len(); + let bytes_len = NonZeroUsize::new(byte_size.max(1)).expect("should be non-zero"); + let builder = + RequestMetadataBuilder::new(data.event_count, data.byte_size, data.json_byte_size); + OtlpGrpcRequest { + signal: make_signal(data.request), + uri: uri.clone(), + dynamic_headers, + finalizers: data.finalizers, + metadata: builder.with_request_size(bytes_len), + } +} + #[cfg(test)] mod tests { use super::*; From 42825ef2e5a6c8467b5e622fcb767c7e4cadcee0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 15:39:58 -0400 Subject: [PATCH 43/68] docs(opentelemetry sink): fix stale nested protocol format in quickstart example --- .../components/sinks/opentelemetry.cue | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/website/cue/reference/components/sinks/opentelemetry.cue b/website/cue/reference/components/sinks/opentelemetry.cue index ffdc1d40b67c2..22d586244c65b 100644 --- a/website/cue/reference/components/sinks/opentelemetry.cue +++ b/website/cue/reference/components/sinks/opentelemetry.cue @@ -106,16 +106,15 @@ components: sinks: opentelemetry: { emit_syslog: inputs: ["remap_syslog"] type: opentelemetry - protocol: - type: http - uri: http://localhost:5318/v1/logs - method: post - encoding: - codec: json - framing: - method: newline_delimited - headers: - content-type: application/json + protocol: http + uri: http://localhost:5318/v1/logs + method: post + encoding: + codec: json + framing: + method: newline_delimited + headers: + content-type: application/json ``` 2. Sample OTEL collector config: From 5f96591a451d9947e9404e2c6b7cd40911dee450 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 17:30:32 -0400 Subject: [PATCH 44/68] perf(opentelemetry sink): clone only the needed tonic client per Service::call --- src/sinks/opentelemetry/grpc.rs | 48 ++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 6b0efd474ac23..de2ba957af33e 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -389,6 +389,15 @@ struct CachedClients { traces: TraceServiceClient, } +/// Holds whichever tonic client is needed for a single `OtlpGrpcRequest`. Only the +/// relevant client is cloned out of `CachedClients`, avoiding two unnecessary clones +/// per `Service::call` invocation. +enum SignalClient { + Logs(LogsServiceClient), + Metrics(MetricsServiceClient), + Traces(TraceServiceClient), +} + #[derive(Clone)] pub struct OtlpGrpcService { /// Tonic clients for the most-recently-seen URI; rebuilt when the rendered URI changes. @@ -419,15 +428,9 @@ impl OtlpGrpcService { } } - /// Returns cloned tonic clients for `uri`, rebuilding them if the URI changed. - fn clients_for( - &mut self, - uri: &Uri, - ) -> ( - LogsServiceClient, - MetricsServiceClient, - TraceServiceClient, - ) { + /// Rebuilds cached tonic clients if the URI changed, then returns the single client + /// matching `signal`. Only that one client is cloned. + fn client_for_signal(&mut self, uri: &Uri, signal: &OtlpSignal) -> SignalClient { if self.clients.as_ref().is_none_or(|c| &c.uri != uri) { let svc = HyperSvc { uri: uri.clone(), @@ -449,7 +452,11 @@ impl OtlpGrpcService { }); } let c = self.clients.as_ref().expect("just populated"); - (c.logs.clone(), c.metrics.clone(), c.traces.clone()) + match signal { + OtlpSignal::Logs(_) => SignalClient::Logs(c.logs.clone()), + OtlpSignal::Metrics(_) => SignalClient::Metrics(c.metrics.clone()), + OtlpSignal::Traces(_) => SignalClient::Traces(c.traces.clone()), + } } } @@ -464,8 +471,8 @@ impl Service for OtlpGrpcService { fn call(&mut self, mut req: OtlpGrpcRequest) -> Self::Future { let (protocol, endpoint) = crate::sinks::util::uri::protocol_endpoint(req.uri.clone()); - // Rebuild clients if the URI changed; clone them out before any `.await`. - let (mut logs_client, mut metrics_client, mut traces_client) = self.clients_for(&req.uri); + // Rebuild clients if the URI changed; clone only the client for this signal type. + let client = self.client_for_signal(&req.uri, &req.signal); let static_headers = std::sync::Arc::clone(&self.headers); let metadata = std::mem::take(req.metadata_mut()); let events_byte_size = metadata.into_events_estimated_json_encoded_byte_size(); @@ -489,25 +496,28 @@ impl Service for OtlpGrpcService { }}; } - let byte_size = match req.signal { - OtlpSignal::Logs(r) => { - let (len, resp) = export!(logs_client, r); + // The signal variant and the client type are always aligned — both are derived + // from `req.signal` in `client_for_signal` — so the mixed arms are unreachable. + let byte_size = match (req.signal, client) { + (OtlpSignal::Logs(r), SignalClient::Logs(mut c)) => { + let (len, resp) = export!(c, r); let rejected = resp.partial_success.map(|ps| ps.rejected_log_records).unwrap_or(0); if rejected > 0 { warn!(rejected, "OTLP collector rejected log records"); } len } - OtlpSignal::Metrics(r) => { - let (len, resp) = export!(metrics_client, r); + (OtlpSignal::Metrics(r), SignalClient::Metrics(mut c)) => { + let (len, resp) = export!(c, r); let rejected = resp.partial_success.map(|ps| ps.rejected_data_points).unwrap_or(0); if rejected > 0 { warn!(rejected, "OTLP collector rejected metric data points"); } len } - OtlpSignal::Traces(r) => { - let (len, resp) = export!(traces_client, r); + (OtlpSignal::Traces(r), SignalClient::Traces(mut c)) => { + let (len, resp) = export!(c, r); let rejected = resp.partial_success.map(|ps| ps.rejected_spans).unwrap_or(0); if rejected > 0 { warn!(rejected, "OTLP collector rejected trace spans"); } len } + _ => unreachable!("signal variant and cached client are always aligned"), }; emit!(EndpointBytesSent { From c3c92b663a6df7b29b05780603a854fb6f4fa93e Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 17:32:59 -0400 Subject: [PATCH 45/68] refactor(grpc): move URI validation to HyperGrpcService construction, eliminate hot-path expect() calls --- src/sinks/opentelemetry/grpc.rs | 7 ++--- src/sinks/util/grpc.rs | 49 +++++++++++++++++++++++---------- src/sinks/vector/service.rs | 5 +--- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index de2ba957af33e..24d6a1fadc0e0 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -261,7 +261,7 @@ async fn grpc_healthcheck( use tonic::Code; use tonic_health::pb::{HealthCheckRequest, health_client::HealthClient}; - let svc = HyperSvc { uri, client }; + let svc = HyperSvc::new(uri, client); let mut health_client = HealthClient::new(svc); let mut req = tonic::Request::new(HealthCheckRequest { @@ -432,10 +432,7 @@ impl OtlpGrpcService { /// matching `signal`. Only that one client is cloned. fn client_for_signal(&mut self, uri: &Uri, signal: &OtlpSignal) -> SignalClient { if self.clients.as_ref().is_none_or(|c| &c.uri != uri) { - let svc = HyperSvc { - uri: uri.clone(), - client: self.hyper_client.clone(), - }; + let svc = HyperSvc::new(uri.clone(), self.hyper_client.clone()); let mut logs = LogsServiceClient::new(svc.clone()); let mut metrics = MetricsServiceClient::new(svc.clone()); let mut traces = TraceServiceClient::new(svc); diff --git a/src/sinks/util/grpc.rs b/src/sinks/util/grpc.rs index 61d32b2fd6871..adfa537973abb 100644 --- a/src/sinks/util/grpc.rs +++ b/src/sinks/util/grpc.rs @@ -1,7 +1,7 @@ use std::task::{Context, Poll}; use futures::future::BoxFuture; -use http::Uri; +use http::{Uri, uri::{Authority, PathAndQuery, Scheme}}; use hyper::client::HttpConnector; use hyper_openssl::HttpsConnector; use hyper_proxy::ProxyConnector; @@ -16,10 +16,36 @@ use tower::Service; /// shared Hyper client. #[derive(Clone, Debug)] pub struct HyperGrpcService { - pub uri: Uri, + scheme: Scheme, + authority: Authority, pub client: hyper::Client>, BoxBody>, } +impl HyperGrpcService { + /// Creates a new [`HyperGrpcService`]. + /// + /// # Panics + /// + /// Panics at construction time if `uri` lacks a scheme or authority. Always supply a + /// URI produced by `with_default_scheme`, which guarantees both components are present. + /// Panicking here (once, at startup) is preferable to panicking inside the hot-path + /// `Service::call` implementation on every request. + pub fn new( + uri: Uri, + client: hyper::Client>, BoxBody>, + ) -> Self { + let scheme = uri + .scheme() + .expect("gRPC service URI must have a scheme — supply a URI from `with_default_scheme`") + .clone(); + let authority = uri + .authority() + .expect("gRPC service URI must have an authority (host:port) — supply a URI from `with_default_scheme`") + .clone(); + Self { scheme, authority, client } + } +} + impl Service> for HyperGrpcService { type Response = hyper::Response; type Error = hyper::Error; @@ -30,24 +56,19 @@ impl Service> for HyperGrpcService { } fn call(&mut self, mut req: hyper::Request) -> Self::Future { - // SAFETY: `self.uri` is always produced by `with_default_scheme` or equivalent, - // which guarantees a scheme and authority. Tonic always sets a path on the request URI. + // scheme and authority are pre-validated at construction; path_and_query falls back + // to "/" if tonic omits it (it never does, but we avoid a panic either way). let uri = Uri::builder() - .scheme(self.uri.scheme().expect("uri always has a scheme").clone()) - .authority( - self.uri - .authority() - .expect("uri always has an authority") - .clone(), - ) + .scheme(self.scheme.clone()) + .authority(self.authority.clone()) .path_and_query( req.uri() .path_and_query() - .expect("tonic request always has a path") - .clone(), + .cloned() + .unwrap_or_else(|| PathAndQuery::from_static("/")), ) .build() - .expect("uri components are always valid"); + .expect("pre-validated scheme and authority always produce a valid URI"); *req.uri_mut() = uri; diff --git a/src/sinks/vector/service.rs b/src/sinks/vector/service.rs index 33c310fcbf321..407a1cf912737 100644 --- a/src/sinks/vector/service.rs +++ b/src/sinks/vector/service.rs @@ -73,10 +73,7 @@ impl VectorService { compression: bool, ) -> Self { let (protocol, endpoint) = uri::protocol_endpoint(uri.clone()); - let mut proto_client = proto_vector::Client::new(HyperSvc { - uri, - client: hyper_client, - }); + let mut proto_client = proto_vector::Client::new(HyperSvc::new(uri, hyper_client)); if compression { proto_client = proto_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); From 21182171e5c9685367c6dcd4a795ede2f297982e Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 27 Mar 2026 17:34:42 -0400 Subject: [PATCH 46/68] =?UTF-8?q?refactor(opentelemetry=20sink):=20reduce?= =?UTF-8?q?=20visibility=20of=20internal=20gRPC=20types=20=E2=80=94=20no?= =?UTF-8?q?=20external=20consumers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sinks/opentelemetry/grpc.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 24d6a1fadc0e0..550a9012bc5bb 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -325,8 +325,7 @@ impl RetryLogic for OtlpGrpcRetryLogic { // ── Error type ─────────────────────────────────────────────────────────────── #[derive(Debug, Snafu)] -#[snafu(visibility(pub))] -pub enum OtlpGrpcError { +enum OtlpGrpcError { #[snafu(display("gRPC request failed: {source}"))] Request { source: tonic::Status }, } @@ -337,16 +336,16 @@ pub enum OtlpGrpcError { /// that retries are atomic: a failed metrics export cannot duplicate a /// previously-accepted logs export. #[derive(Clone)] -pub struct OtlpGrpcRequest { - pub signal: OtlpSignal, - pub uri: Uri, +struct OtlpGrpcRequest { + signal: OtlpSignal, + uri: Uri, /// Per-event rendered values for dynamic (templated) metadata headers. - pub dynamic_headers: Vec<( + dynamic_headers: Vec<( tonic::metadata::AsciiMetadataKey, tonic::metadata::AsciiMetadataValue, )>, - pub finalizers: EventFinalizers, - pub metadata: RequestMetadata, + finalizers: EventFinalizers, + metadata: RequestMetadata, } impl Finalizable for OtlpGrpcRequest { @@ -365,7 +364,7 @@ impl MetaDescriptive for OtlpGrpcRequest { } } -pub struct OtlpGrpcResponse { +struct OtlpGrpcResponse { events_byte_size: GroupedCountByteSize, } @@ -399,7 +398,7 @@ enum SignalClient { } #[derive(Clone)] -pub struct OtlpGrpcService { +struct OtlpGrpcService { /// Tonic clients for the most-recently-seen URI; rebuilt when the rendered URI changes. /// Each Tower concurrency-slot clone owns its cache independently — no shared mutex needed. clients: Option, @@ -412,7 +411,7 @@ pub struct OtlpGrpcService { } impl OtlpGrpcService { - pub fn new( + fn new( hyper_client: hyper::Client>, BoxBody>, compression: bool, headers: Vec<( @@ -592,7 +591,7 @@ impl ByteSizeOf for OtlpEventData { /// request payload sent via gRPC after batching. Each variant corresponds to one signal /// type so that requests can be retried independently. #[derive(Clone)] -pub enum OtlpSignal { +enum OtlpSignal { Logs(ExportLogsServiceRequest), Metrics(ExportMetricsServiceRequest), Traces(ExportTraceServiceRequest), @@ -673,9 +672,9 @@ impl OtlpBatch { } } -pub struct OtlpGrpcSink { - pub batch_settings: BatcherSettings, - pub service: S, +struct OtlpGrpcSink { + batch_settings: BatcherSettings, + service: S, uri_template: Template, use_https: bool, dynamic_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)>, From 94a226d4c18f63cb0228ae417f2a25283880177f Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 10:12:38 -0400 Subject: [PATCH 47/68] perf(opentelemetry sink): use cached byte_size in ByteSizeOf::allocated_bytes, drop dead encoded_len wrapper --- src/sinks/opentelemetry/grpc.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 550a9012bc5bb..50c1d9223a6dd 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -583,7 +583,7 @@ impl ByteSizeOf for OtlpEventData { } fn allocated_bytes(&self) -> usize { - self.signal.encoded_len() + self.byte_size } } @@ -597,15 +597,6 @@ enum OtlpSignal { Traces(ExportTraceServiceRequest), } -impl OtlpSignal { - fn encoded_len(&self) -> usize { - match self { - OtlpSignal::Logs(r) => r.encoded_len(), - OtlpSignal::Metrics(r) => r.encoded_len(), - OtlpSignal::Traces(r) => r.encoded_len(), - } - } -} /// Per-signal accumulator tracking the merged proto request and its associated /// event metadata. Kept separate so each signal can be retried independently. From 739049907d5147e64b242eac8f451cfe213ec98b Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 10:48:26 -0400 Subject: [PATCH 48/68] fix(opentelemetry sink): include error_message in OTLP partial-success warnings for all signal types --- src/sinks/opentelemetry/grpc.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 50c1d9223a6dd..b5e8d9c3618b9 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -497,20 +497,29 @@ impl Service for OtlpGrpcService { let byte_size = match (req.signal, client) { (OtlpSignal::Logs(r), SignalClient::Logs(mut c)) => { let (len, resp) = export!(c, r); - let rejected = resp.partial_success.map(|ps| ps.rejected_log_records).unwrap_or(0); - if rejected > 0 { warn!(rejected, "OTLP collector rejected log records"); } + if let Some(ps) = resp.partial_success { + if ps.rejected_log_records > 0 { + warn!(rejected = ps.rejected_log_records, message = ps.error_message, "OTLP collector rejected log records"); + } + } len } (OtlpSignal::Metrics(r), SignalClient::Metrics(mut c)) => { let (len, resp) = export!(c, r); - let rejected = resp.partial_success.map(|ps| ps.rejected_data_points).unwrap_or(0); - if rejected > 0 { warn!(rejected, "OTLP collector rejected metric data points"); } + if let Some(ps) = resp.partial_success { + if ps.rejected_data_points > 0 { + warn!(rejected = ps.rejected_data_points, message = ps.error_message, "OTLP collector rejected metric data points"); + } + } len } (OtlpSignal::Traces(r), SignalClient::Traces(mut c)) => { let (len, resp) = export!(c, r); - let rejected = resp.partial_success.map(|ps| ps.rejected_spans).unwrap_or(0); - if rejected > 0 { warn!(rejected, "OTLP collector rejected trace spans"); } + if let Some(ps) = resp.partial_success { + if ps.rejected_spans > 0 { + warn!(rejected = ps.rejected_spans, message = ps.error_message, "OTLP collector rejected trace spans"); + } + } len } _ => unreachable!("signal variant and cached client are always aligned"), From 000520dbbe9a1c0e5c5cd5ad9fc5cbc2115b719e Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 12:43:04 -0400 Subject: [PATCH 49/68] refactor(opentelemetry sink): collapse partial-success let-chains and inline signal detection into encode_event --- src/sinks/opentelemetry/grpc.rs | 84 +++++++++------------------------ 1 file changed, 22 insertions(+), 62 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index b5e8d9c3618b9..61539b3b64e6a 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -497,28 +497,22 @@ impl Service for OtlpGrpcService { let byte_size = match (req.signal, client) { (OtlpSignal::Logs(r), SignalClient::Logs(mut c)) => { let (len, resp) = export!(c, r); - if let Some(ps) = resp.partial_success { - if ps.rejected_log_records > 0 { - warn!(rejected = ps.rejected_log_records, message = ps.error_message, "OTLP collector rejected log records"); - } + if let Some(ps) = resp.partial_success && ps.rejected_log_records > 0 { + warn!(rejected = ps.rejected_log_records, message = ps.error_message, "OTLP collector rejected log records"); } len } (OtlpSignal::Metrics(r), SignalClient::Metrics(mut c)) => { let (len, resp) = export!(c, r); - if let Some(ps) = resp.partial_success { - if ps.rejected_data_points > 0 { - warn!(rejected = ps.rejected_data_points, message = ps.error_message, "OTLP collector rejected metric data points"); - } + if let Some(ps) = resp.partial_success && ps.rejected_data_points > 0 { + warn!(rejected = ps.rejected_data_points, message = ps.error_message, "OTLP collector rejected metric data points"); } len } (OtlpSignal::Traces(r), SignalClient::Traces(mut c)) => { let (len, resp) = export!(c, r); - if let Some(ps) = resp.partial_success { - if ps.rejected_spans > 0 { - warn!(rejected = ps.rejected_spans, message = ps.error_message, "OTLP collector rejected trace spans"); - } + if let Some(ps) = resp.partial_success && ps.rejected_spans > 0 { + warn!(rejected = ps.rejected_spans, message = ps.error_message, "OTLP collector rejected trace spans"); } len } @@ -857,63 +851,29 @@ fn encode_event( serializer: &mut OtlpSerializer, event: &Event, ) -> Result, vector_common::Error> { - let signal_type = detect_signal_type(event); - let signal_type = match signal_type { - Some(t) => t, - None => return Ok(None), - }; - let mut buf = BytesMut::new(); // Clone the event to pass ownership to the encoder while keeping the original // for metadata extraction in the caller. The clone is shallow for logs/traces. - serializer.encode(event.clone(), &mut buf)?; - - let bytes = buf.freeze(); - match signal_type { - SignalType::Logs => { - let req = ExportLogsServiceRequest::decode(bytes)?; - Ok(Some(OtlpSignal::Logs(req))) - } - SignalType::Metrics => { - let req = ExportMetricsServiceRequest::decode(bytes)?; - Ok(Some(OtlpSignal::Metrics(req))) + match event { + Event::Log(log) if log.contains(RESOURCE_LOGS_JSON_FIELD) => { + serializer.encode(event.clone(), &mut buf)?; + Ok(Some(OtlpSignal::Logs(ExportLogsServiceRequest::decode(buf.freeze())?))) } - SignalType::Traces => { - let req = ExportTraceServiceRequest::decode(bytes)?; - Ok(Some(OtlpSignal::Traces(req))) + Event::Log(log) if log.contains(RESOURCE_METRICS_JSON_FIELD) => { + serializer.encode(event.clone(), &mut buf)?; + Ok(Some(OtlpSignal::Metrics(ExportMetricsServiceRequest::decode(buf.freeze())?))) } - } -} - -enum SignalType { - Logs, - Metrics, - Traces, -} - -fn detect_signal_type(event: &Event) -> Option { - match event { - Event::Log(log) => { - if log.contains(RESOURCE_LOGS_JSON_FIELD) { - Some(SignalType::Logs) - } else if log.contains(RESOURCE_METRICS_JSON_FIELD) { - Some(SignalType::Metrics) - } else if log.contains(RESOURCE_SPANS_JSON_FIELD) { - // OTLP spans can arrive as Log events when the source does not use - // use_otlp_decoding.traces = true. - Some(SignalType::Traces) - } else { - None - } + // OTLP spans can arrive as Log events when the source does not use + // use_otlp_decoding.traces = true. + Event::Log(log) if log.contains(RESOURCE_SPANS_JSON_FIELD) => { + serializer.encode(event.clone(), &mut buf)?; + Ok(Some(OtlpSignal::Traces(ExportTraceServiceRequest::decode(buf.freeze())?))) } - Event::Trace(trace) => { - if trace.contains(RESOURCE_SPANS_JSON_FIELD) { - Some(SignalType::Traces) - } else { - None - } + Event::Trace(trace) if trace.contains(RESOURCE_SPANS_JSON_FIELD) => { + serializer.encode(event.clone(), &mut buf)?; + Ok(Some(OtlpSignal::Traces(ExportTraceServiceRequest::decode(buf.freeze())?))) } - Event::Metric(_) => None, // Native Vector metrics not supported by OtlpSerializer + _ => Ok(None), } } From dd97e1401b8c4cf3b48d79a50efc4dfe2d707e51 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 12:46:11 -0400 Subject: [PATCH 50/68] fix(opentelemetry sink): align gRPC batch config type with byte-size runtime behavior --- src/sinks/opentelemetry/grpc.rs | 4 ++-- src/sinks/opentelemetry/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 61539b3b64e6a..e868ec98fc5cd 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -49,7 +49,7 @@ use crate::{ sinks::{ Healthcheck, VectorSink, util::{ - BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, + BatchConfig, RealtimeSizeBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, StreamSink, http::RequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, }, }, @@ -110,7 +110,7 @@ pub(super) struct GrpcSinkConfig { #[configurable(derived)] #[serde(default)] - pub batch: BatchConfig, + pub batch: BatchConfig, #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 3f450e95fff67..3f38998301efc 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -15,8 +15,8 @@ use crate::{ Healthcheck, VectorSink, http::config::{HttpMethod, HttpSinkConfig}, util::{ - BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, - RealtimeSizeBasedDefaultBatchSettings, http::RequestConfig, + BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, + http::RequestConfig, }, }, template::Template, @@ -64,7 +64,7 @@ pub enum OtlpProtocol { Grpc { #[configurable(derived)] #[serde(default)] - batch: BatchConfig, + batch: BatchConfig, }, } From 1b092d03f131973bdb95f644c1dbd0974950bd8d Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 12:48:28 -0400 Subject: [PATCH 51/68] refactor(grpc): consolidate with_default_scheme into sinks/util/grpc, remove duplicate implementations --- src/sinks/opentelemetry/grpc.rs | 20 +++---------------- src/sinks/util/grpc.rs | 18 +++++++++++++++++ src/sinks/vector/config.rs | 35 +-------------------------------- src/sinks/vector/mod.rs | 7 ++++--- 4 files changed, 26 insertions(+), 54 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index e868ec98fc5cd..8a85ae49ce167 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -7,10 +7,7 @@ use std::{ use async_trait::async_trait; use bytes::BytesMut; use futures::{StreamExt, TryFutureExt, future::BoxFuture, stream::BoxStream}; -use http::{ - Uri, - uri::{PathAndQuery, Scheme}, -}; +use http::Uri; use hyper::client::HttpConnector; use hyper_openssl::HttpsConnector; use hyper_proxy::ProxyConnector; @@ -42,7 +39,7 @@ use vector_lib::{ use crate::{ config::{AcknowledgementsConfig, SinkContext, SinkHealthcheckOptions}, - sinks::util::grpc::HyperGrpcService, + sinks::util::grpc::{HyperGrpcService, with_default_scheme}, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, internal_events::{EndpointBytesSent, SinkRequestBuildError}, @@ -72,18 +69,6 @@ pub enum GrpcCompression { Gzip, } -pub(super) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { - if uri.scheme().is_none() { - let mut parts = uri.into_parts(); - parts.scheme = Some(if tls { Scheme::HTTPS } else { Scheme::HTTP }); - if parts.path_and_query.is_none() { - parts.path_and_query = Some(PathAndQuery::from_static("/")); - } - Ok(Uri::from_parts(parts)?) - } else { - Ok(uri) - } -} /// Configuration for the OpenTelemetry sink's gRPC transport. #[configurable_component] @@ -918,6 +903,7 @@ fn signal_into_request( #[cfg(test)] mod tests { use super::*; + use crate::sinks::util::grpc::with_default_scheme; #[test] fn generate_grpc_config() { diff --git a/src/sinks/util/grpc.rs b/src/sinks/util/grpc.rs index adfa537973abb..29c3e159bb54b 100644 --- a/src/sinks/util/grpc.rs +++ b/src/sinks/util/grpc.rs @@ -8,6 +8,24 @@ use hyper_proxy::ProxyConnector; use tonic::body::BoxBody; use tower::Service; +/// Adds a default scheme to a URI that lacks one. +/// +/// Returns the URI unchanged if a scheme is already present. Otherwise prepends +/// `https` when `tls` is `true`, or `http` when `false`. Also sets the +/// path-and-query to `/` if missing. +pub(crate) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { + if uri.scheme().is_none() { + let mut parts = uri.into_parts(); + parts.scheme = Some(if tls { Scheme::HTTPS } else { Scheme::HTTP }); + if parts.path_and_query.is_none() { + parts.path_and_query = Some(PathAndQuery::from_static("/")); + } + Ok(Uri::from_parts(parts)?) + } else { + Ok(uri) + } +} + /// A Tower [`Service`] that routes gRPC requests through a Hyper HTTP/2 client, /// substituting the scheme and authority from a fixed base URI while preserving /// the path/query set by tonic. diff --git a/src/sinks/vector/config.rs b/src/sinks/vector/config.rs index 52601115cb67a..b182a6a2d481f 100644 --- a/src/sinks/vector/config.rs +++ b/src/sinks/vector/config.rs @@ -110,7 +110,7 @@ fn default_config(address: &str) -> VectorConfig { impl SinkConfig for VectorConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSinkType, Healthcheck)> { let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), false)?; - let uri = with_default_scheme(&self.address, tls.is_tls())?; + let uri = crate::sinks::util::grpc::with_default_scheme(self.address.parse()?, tls.is_tls())?; let client = new_client(&tls, cx.proxy())?; @@ -176,39 +176,6 @@ async fn healthcheck( } } -/// grpc doesn't like an address without a scheme, so we default to http or https if one isn't -/// specified in the address. -pub fn with_default_scheme(address: &str, tls: bool) -> crate::Result { - let uri: Uri = address.parse()?; - if uri.scheme().is_none() { - // Default the scheme to http or https. - let mut parts = uri.into_parts(); - - parts.scheme = if tls { - Some( - "https" - .parse() - .unwrap_or_else(|_| unreachable!("https should be valid")), - ) - } else { - Some( - "http" - .parse() - .unwrap_or_else(|_| unreachable!("http should be valid")), - ) - }; - - if parts.path_and_query.is_none() { - parts.path_and_query = Some( - "/".parse() - .unwrap_or_else(|_| unreachable!("root should be valid")), - ); - } - Ok(Uri::from_parts(parts)?) - } else { - Ok(uri) - } -} fn new_client( tls_settings: &MaybeTlsSettings, diff --git a/src/sinks/vector/mod.rs b/src/sinks/vector/mod.rs index 4cf882b1d2ee6..08c1f1ae87b8c 100644 --- a/src/sinks/vector/mod.rs +++ b/src/sinks/vector/mod.rs @@ -49,7 +49,8 @@ mod tests { event::{BatchNotifier, BatchStatus}, }; - use super::{config::with_default_scheme, *}; + use super::*; + use crate::sinks::util::grpc::with_default_scheme; use crate::{ config::{SinkConfig as _, SinkContext}, event::Event, @@ -183,11 +184,11 @@ mod tests { #[test] fn test_with_default_scheme() { assert_eq!( - with_default_scheme("0.0.0.0", false).unwrap().to_string(), + with_default_scheme("0.0.0.0".parse().unwrap(), false).unwrap().to_string(), "http://0.0.0.0/" ); assert_eq!( - with_default_scheme("0.0.0.0", true).unwrap().to_string(), + with_default_scheme("0.0.0.0".parse().unwrap(), true).unwrap().to_string(), "https://0.0.0.0/" ); } From cb501e4f0295b0c6fe1489c1c5ea3bd61ada43aa Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 12:58:37 -0400 Subject: [PATCH 52/68] test(opentelemetry sink): use OpenTelemetryConfig in integration tests to exercise the full config path --- src/sinks/opentelemetry/integration_tests.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs index 45993ff0acf28..24c53e96b294f 100644 --- a/src/sinks/opentelemetry/integration_tests.rs +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -10,9 +10,9 @@ use vector_lib::{ }, }; -use super::grpc::GrpcSinkConfig; +use super::OpenTelemetryConfig; use crate::{ - config::SinkContext, + config::{SinkConfig as _, SinkContext}, test_util::{ components::{HTTP_SINK_TAGS, run_and_assert_sink_compliance}, wait_for_tcp, @@ -70,8 +70,9 @@ async fn delivers_logs_via_grpc() { let address = sink_grpc_address(); wait_for_tcp(address.clone()).await; - let config: GrpcSinkConfig = toml::from_str(&format!( + let config: OpenTelemetryConfig = toml::from_str(&format!( r#" + protocol = "grpc" uri = "http://{address}" "# )) @@ -90,8 +91,9 @@ async fn delivers_logs_via_grpc_template_uri() { let host = sink_grpc_address(); wait_for_tcp(host.clone()).await; - let config: GrpcSinkConfig = toml::from_str( + let config: OpenTelemetryConfig = toml::from_str( r#" + protocol = "grpc" uri = "http://{{ host }}:4317" "#, ) From 0cf6564a59cb8b0a9bd0d721549dbb43c9ae1785 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 13:03:59 -0400 Subject: [PATCH 53/68] Revert "fix(opentelemetry sink): align gRPC batch config type with byte-size runtime behavior" This reverts commit dd97e1401b8c4cf3b48d79a50efc4dfe2d707e51. --- src/sinks/opentelemetry/grpc.rs | 4 ++-- src/sinks/opentelemetry/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 8a85ae49ce167..91eeb9bf9445d 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -46,7 +46,7 @@ use crate::{ sinks::{ Healthcheck, VectorSink, util::{ - BatchConfig, RealtimeSizeBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, + BatchConfig, RealtimeEventBasedDefaultBatchSettings, ServiceBuilderExt, SinkBuilderExt, StreamSink, http::RequestConfig, metadata::RequestMetadataBuilder, retries::RetryLogic, }, }, @@ -95,7 +95,7 @@ pub(super) struct GrpcSinkConfig { #[configurable(derived)] #[serde(default)] - pub batch: BatchConfig, + pub batch: BatchConfig, #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 3f38998301efc..3f450e95fff67 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -15,8 +15,8 @@ use crate::{ Healthcheck, VectorSink, http::config::{HttpMethod, HttpSinkConfig}, util::{ - BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, - http::RequestConfig, + BatchConfig, Compression, RealtimeEventBasedDefaultBatchSettings, + RealtimeSizeBasedDefaultBatchSettings, http::RequestConfig, }, }, template::Template, @@ -64,7 +64,7 @@ pub enum OtlpProtocol { Grpc { #[configurable(derived)] #[serde(default)] - batch: BatchConfig, + batch: BatchConfig, }, } From 18e0eb599311c6bf007ef8c1fc9dd4cd9bd9ffa7 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 13:06:12 -0400 Subject: [PATCH 54/68] docs(opentelemetry sink): document intentional use of event-based batch settings with byte-size flush --- src/sinks/opentelemetry/grpc.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 91eeb9bf9445d..255e3d1545c40 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -774,6 +774,12 @@ where // Partition by (rendered URI, dynamic headers) so that events destined for // different collectors or with different tenant metadata are never merged into // the same batch. Each partition is independently flushed by size or timeout. + // + // `as_byte_size_config` is intentional here: gRPC batches are bounded by the + // serialized protobuf byte size (via `ByteSizeOf::allocated_bytes`), not by raw + // event count. `RealtimeEventBasedDefaultBatchSettings` is kept as the config + // type so that the default `max_events = 1000` cap is preserved for throughput; + // the byte-size limit is then the primary flush trigger. .batched_partitioned( GrpcPartitioner, self.batch_settings.timeout, From 54ef34d3a6b0724cbe4d5aad25ad6cc9fb0d2ab1 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 13:37:16 -0400 Subject: [PATCH 55/68] =?UTF-8?q?docs(opentelemetry=20sink):=20correct=20b?= =?UTF-8?q?atch=20settings=20comment=20=E2=80=94=20event=20count=20is=20pr?= =?UTF-8?q?imary=20flush=20trigger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sinks/opentelemetry/grpc.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 255e3d1545c40..ae6b50334c779 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -775,11 +775,12 @@ where // different collectors or with different tenant metadata are never merged into // the same batch. Each partition is independently flushed by size or timeout. // - // `as_byte_size_config` is intentional here: gRPC batches are bounded by the - // serialized protobuf byte size (via `ByteSizeOf::allocated_bytes`), not by raw - // event count. `RealtimeEventBasedDefaultBatchSettings` is kept as the config - // type so that the default `max_events = 1000` cap is preserved for throughput; - // the byte-size limit is then the primary flush trigger. + // `as_byte_size_config` is intentional here: the primary flush triggers are the + // event-count limit (default 1000, from `RealtimeEventBasedDefaultBatchSettings`) + // and the 1s timeout. The byte limit defaults to `usize::MAX` (effectively + // disabled) because `MAX_BYTES = None` for that settings type. `as_byte_size_config` + // selects `ByteSizeOf` as the size metric so that an operator-configured + // `batch.max_bytes` is measured consistently with the rest of Vector. .batched_partitioned( GrpcPartitioner, self.batch_settings.timeout, From 908f733ebd9cd608686f6c27c4ce1ea53282e335 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:19:10 -0400 Subject: [PATCH 56/68] fix(opentelemetry sink): honour cx.healthcheck.uri override in gRPC healthcheck --- src/sinks/opentelemetry/grpc.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index ae6b50334c779..e7700985852db 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -186,9 +186,16 @@ impl GrpcSinkConfig { } let client = new_grpc_client(&tls, cx.proxy())?; + let healthcheck_uri = cx + .healthcheck + .uri + .clone() + .map(|u| with_default_scheme(u.uri, use_https)) + .transpose()? + .or(static_uri.clone()); let healthcheck = Box::pin(grpc_healthcheck( client.clone(), - static_uri, + healthcheck_uri, static_grpc_headers.clone(), cx.healthcheck, )); From c00904d9e461f602b9126f0b498f2833a4536061 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:25:42 -0400 Subject: [PATCH 57/68] emit dropped events metric when dropping non-OTLP formatted events --- src/sinks/opentelemetry/grpc.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index e7700985852db..0768db27a0d11 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -491,6 +491,10 @@ impl Service for OtlpGrpcService { let (len, resp) = export!(c, r); if let Some(ps) = resp.partial_success && ps.rejected_log_records > 0 { warn!(rejected = ps.rejected_log_records, message = ps.error_message, "OTLP collector rejected log records"); + emit!(ComponentEventsDropped:: { + count: ps.rejected_log_records as usize, + reason: if ps.error_message.is_empty() { "OTLP partial success rejection" } else { &ps.error_message }, + }); } len } @@ -498,6 +502,10 @@ impl Service for OtlpGrpcService { let (len, resp) = export!(c, r); if let Some(ps) = resp.partial_success && ps.rejected_data_points > 0 { warn!(rejected = ps.rejected_data_points, message = ps.error_message, "OTLP collector rejected metric data points"); + emit!(ComponentEventsDropped:: { + count: ps.rejected_data_points as usize, + reason: if ps.error_message.is_empty() { "OTLP partial success rejection" } else { &ps.error_message }, + }); } len } @@ -505,6 +513,10 @@ impl Service for OtlpGrpcService { let (len, resp) = export!(c, r); if let Some(ps) = resp.partial_success && ps.rejected_spans > 0 { warn!(rejected = ps.rejected_spans, message = ps.error_message, "OTLP collector rejected trace spans"); + emit!(ComponentEventsDropped:: { + count: ps.rejected_spans as usize, + reason: if ps.error_message.is_empty() { "OTLP partial success rejection" } else { &ps.error_message }, + }); } len } From 23bc1045854a033adc811e2d8ee24c5fdaba5531 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:39:06 -0400 Subject: [PATCH 58/68] fix(opentelemetry sink): mark dropped events as Rejected and propagate partial-success failures --- src/sinks/opentelemetry/grpc.rs | 183 ++++++++++++++++++++++++-------- src/sinks/opentelemetry/mod.rs | 13 ++- src/sinks/util/grpc.rs | 11 +- src/sinks/vector/config.rs | 4 +- src/sinks/vector/mod.rs | 8 +- 5 files changed, 163 insertions(+), 56 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 0768db27a0d11..89cee5a43012c 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -39,10 +39,10 @@ use vector_lib::{ use crate::{ config::{AcknowledgementsConfig, SinkContext, SinkHealthcheckOptions}, - sinks::util::grpc::{HyperGrpcService, with_default_scheme}, event::{Event, EventFinalizers, EventStatus, Finalizable}, http::build_proxy_connector, internal_events::{EndpointBytesSent, SinkRequestBuildError}, + sinks::util::grpc::{HyperGrpcService, with_default_scheme}, sinks::{ Healthcheck, VectorSink, util::{ @@ -69,7 +69,6 @@ pub enum GrpcCompression { Gzip, } - /// Configuration for the OpenTelemetry sink's gRPC transport. #[configurable_component] #[derive(Clone, Debug)] @@ -218,7 +217,6 @@ impl GrpcSinkConfig { Ok((VectorSink::from_event_streamsink(sink), healthcheck)) } - } fn new_grpc_client( @@ -358,11 +356,20 @@ impl MetaDescriptive for OtlpGrpcRequest { struct OtlpGrpcResponse { events_byte_size: GroupedCountByteSize, + /// True when the collector returned a partial-success response with one or more rejected + /// records. The entire batch is marked `Rejected` so that upstream sources are notified of + /// data loss. This may cause the accepted portion to be re-sent if the source retries, but + /// silent data loss is worse than potential duplication. + had_partial_success: bool, } impl DriverResponse for OtlpGrpcResponse { fn event_status(&self) -> EventStatus { - EventStatus::Delivered + if self.had_partial_success { + EventStatus::Rejected + } else { + EventStatus::Delivered + } } fn events_sent(&self) -> &GroupedCountByteSize { @@ -396,10 +403,12 @@ struct OtlpGrpcService { clients: Option, hyper_client: hyper::Client>, BoxBody>, compression: bool, - headers: std::sync::Arc>, + headers: std::sync::Arc< + Vec<( + tonic::metadata::AsciiMetadataKey, + tonic::metadata::AsciiMetadataValue, + )>, + >, } impl OtlpGrpcService { @@ -486,39 +495,78 @@ impl Service for OtlpGrpcService { // The signal variant and the client type are always aligned — both are derived // from `req.signal` in `client_for_signal` — so the mixed arms are unreachable. - let byte_size = match (req.signal, client) { + let (byte_size, had_partial_success) = match (req.signal, client) { (OtlpSignal::Logs(r), SignalClient::Logs(mut c)) => { let (len, resp) = export!(c, r); - if let Some(ps) = resp.partial_success && ps.rejected_log_records > 0 { - warn!(rejected = ps.rejected_log_records, message = ps.error_message, "OTLP collector rejected log records"); + let partial = if let Some(ps) = resp.partial_success + && ps.rejected_log_records > 0 + { + warn!( + rejected = ps.rejected_log_records, + message = ps.error_message, + "OTLP collector rejected log records" + ); emit!(ComponentEventsDropped:: { count: ps.rejected_log_records as usize, - reason: if ps.error_message.is_empty() { "OTLP partial success rejection" } else { &ps.error_message }, + reason: if ps.error_message.is_empty() { + "OTLP partial success rejection" + } else { + &ps.error_message + }, }); - } - len + true + } else { + false + }; + (len, partial) } (OtlpSignal::Metrics(r), SignalClient::Metrics(mut c)) => { let (len, resp) = export!(c, r); - if let Some(ps) = resp.partial_success && ps.rejected_data_points > 0 { - warn!(rejected = ps.rejected_data_points, message = ps.error_message, "OTLP collector rejected metric data points"); + let partial = if let Some(ps) = resp.partial_success + && ps.rejected_data_points > 0 + { + warn!( + rejected = ps.rejected_data_points, + message = ps.error_message, + "OTLP collector rejected metric data points" + ); emit!(ComponentEventsDropped:: { count: ps.rejected_data_points as usize, - reason: if ps.error_message.is_empty() { "OTLP partial success rejection" } else { &ps.error_message }, + reason: if ps.error_message.is_empty() { + "OTLP partial success rejection" + } else { + &ps.error_message + }, }); - } - len + true + } else { + false + }; + (len, partial) } (OtlpSignal::Traces(r), SignalClient::Traces(mut c)) => { let (len, resp) = export!(c, r); - if let Some(ps) = resp.partial_success && ps.rejected_spans > 0 { - warn!(rejected = ps.rejected_spans, message = ps.error_message, "OTLP collector rejected trace spans"); + let partial = if let Some(ps) = resp.partial_success + && ps.rejected_spans > 0 + { + warn!( + rejected = ps.rejected_spans, + message = ps.error_message, + "OTLP collector rejected trace spans" + ); emit!(ComponentEventsDropped:: { count: ps.rejected_spans as usize, - reason: if ps.error_message.is_empty() { "OTLP partial success rejection" } else { &ps.error_message }, + reason: if ps.error_message.is_empty() { + "OTLP partial success rejection" + } else { + &ps.error_message + }, }); - } - len + true + } else { + false + }; + (len, partial) } _ => unreachable!("signal variant and cached client are always aligned"), }; @@ -529,7 +577,10 @@ impl Service for OtlpGrpcService { endpoint: &endpoint, }); - Ok(OtlpGrpcResponse { events_byte_size }) + Ok(OtlpGrpcResponse { + events_byte_size, + had_partial_success, + }) }; Box::pin(future.map_err(|err: OtlpGrpcError| -> crate::Error { Box::new(err) })) @@ -604,7 +655,6 @@ enum OtlpSignal { Traces(ExportTraceServiceRequest), } - /// Per-signal accumulator tracking the merged proto request and its associated /// event metadata. Kept separate so each signal can be retried independently. struct SignalData { @@ -622,7 +672,13 @@ impl SignalData { byte_size: usize, json_byte_size: GroupedCountByteSize, ) -> Self { - SignalData { request, finalizers, event_count: 1, byte_size, json_byte_size } + SignalData { + request, + finalizers, + event_count: 1, + byte_size, + json_byte_size, + } } fn merge( @@ -652,18 +708,37 @@ struct OtlpBatch { impl OtlpBatch { fn push(&mut self, item: OtlpEventData) { - let OtlpEventData { byte_size, json_byte_size, finalizers, signal, uri: _, dynamic_headers: _ } = item; + let OtlpEventData { + byte_size, + json_byte_size, + finalizers, + signal, + uri: _, + dynamic_headers: _, + } = item; match signal { OtlpSignal::Logs(req) => match &mut self.logs { - Some(existing) => existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| e.resource_logs.extend(r.resource_logs)), + Some(existing) => { + existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| { + e.resource_logs.extend(r.resource_logs) + }) + } slot => *slot = Some(SignalData::new(req, finalizers, byte_size, json_byte_size)), }, OtlpSignal::Metrics(req) => match &mut self.metrics { - Some(existing) => existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| e.resource_metrics.extend(r.resource_metrics)), + Some(existing) => { + existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| { + e.resource_metrics.extend(r.resource_metrics) + }) + } slot => *slot = Some(SignalData::new(req, finalizers, byte_size, json_byte_size)), }, OtlpSignal::Traces(req) => match &mut self.traces { - Some(existing) => existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| e.resource_spans.extend(r.resource_spans)), + Some(existing) => { + existing.merge(req, finalizers, byte_size, json_byte_size, |e, r| { + e.resource_spans.extend(r.resource_spans) + }) + } slot => *slot = Some(SignalData::new(req, finalizers, byte_size, json_byte_size)), }, } @@ -698,15 +773,15 @@ where .filter_map(move |mut event| { let rendered = match uri_template.render_string(&event) { Ok(s) => s, - Err(e) => return reject_event(format!("failed to render gRPC URI template: {e}")), + Err(e) => return reject_event(event.take_finalizers(), format!("failed to render gRPC URI template: {e}")), }; let parsed = match rendered.parse::() { Ok(u) => u, - Err(e) => return reject_event(format!("failed to parse rendered gRPC URI: {e}")), + Err(e) => return reject_event(event.take_finalizers(), format!("failed to parse rendered gRPC URI: {e}")), }; let u = match with_default_scheme(parsed, use_https) { Ok(u) => u, - Err(e) => return reject_event(format!("invalid gRPC URI after rendering template: {e}")), + Err(e) => return reject_event(event.take_finalizers(), format!("invalid gRPC URI after rendering template: {e}")), }; let uri = match u.scheme_str() { Some("https") if !use_https => { @@ -717,6 +792,7 @@ where // surface a clear error so the operator can add `tls:` or // use a static `https://` scheme prefix. return reject_event( + event.take_finalizers(), "rendered gRPC URI uses \"https\" but the sink \ has no TLS connector; add a `tls:` block or use \ a static \"https://\" URI prefix so TLS is \ @@ -725,7 +801,7 @@ where } Some("http") | Some("https") => u, other => { - return reject_event(format!( + return reject_event(event.take_finalizers(), format!( "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", other.unwrap_or("") )); @@ -743,7 +819,7 @@ where match tonic::metadata::AsciiMetadataValue::try_from(rendered.as_str()) { Ok(value) => dynamic_headers.push((key.clone(), value)), Err(e) => { - return reject_event(format!( + return reject_event(event.take_finalizers(), format!( "gRPC metadata value for key {:?} is not valid ASCII: {e}", key.as_str() )); @@ -751,7 +827,7 @@ where } } Err(e) => { - return reject_event(format!( + return reject_event(event.take_finalizers(), format!( "failed to render gRPC metadata template for key {:?}: {e}", key.as_str() )); @@ -783,11 +859,12 @@ where // OTLP-structured events; plain log events from non-OTLP sources // must be encoded with an OTLP codec before reaching this sink. reject_event( + event.take_finalizers(), "event is not OTLP-encoded (missing resourceLogs, resourceMetrics, \ or resourceSpans field); this sink only accepts OTLP-structured events", ) } - Err(e) => reject_event(e.to_string()), + Err(e) => reject_event(event.take_finalizers(), e.to_string()), } }) // Partition by (rendered URI, dynamic headers) so that events destined for @@ -868,21 +945,29 @@ fn encode_event( match event { Event::Log(log) if log.contains(RESOURCE_LOGS_JSON_FIELD) => { serializer.encode(event.clone(), &mut buf)?; - Ok(Some(OtlpSignal::Logs(ExportLogsServiceRequest::decode(buf.freeze())?))) + Ok(Some(OtlpSignal::Logs(ExportLogsServiceRequest::decode( + buf.freeze(), + )?))) } Event::Log(log) if log.contains(RESOURCE_METRICS_JSON_FIELD) => { serializer.encode(event.clone(), &mut buf)?; - Ok(Some(OtlpSignal::Metrics(ExportMetricsServiceRequest::decode(buf.freeze())?))) + Ok(Some(OtlpSignal::Metrics( + ExportMetricsServiceRequest::decode(buf.freeze())?, + ))) } // OTLP spans can arrive as Log events when the source does not use // use_otlp_decoding.traces = true. Event::Log(log) if log.contains(RESOURCE_SPANS_JSON_FIELD) => { serializer.encode(event.clone(), &mut buf)?; - Ok(Some(OtlpSignal::Traces(ExportTraceServiceRequest::decode(buf.freeze())?))) + Ok(Some(OtlpSignal::Traces(ExportTraceServiceRequest::decode( + buf.freeze(), + )?))) } Event::Trace(trace) if trace.contains(RESOURCE_SPANS_JSON_FIELD) => { serializer.encode(event.clone(), &mut buf)?; - Ok(Some(OtlpSignal::Traces(ExportTraceServiceRequest::decode(buf.freeze())?))) + Ok(Some(OtlpSignal::Traces(ExportTraceServiceRequest::decode( + buf.freeze(), + )?))) } _ => Ok(None), } @@ -890,11 +975,17 @@ fn encode_event( /// Drop an event that could not be prepared for export. /// -/// Emits [`SinkRequestBuildError`] and [`ComponentEventsDropped`], then returns -/// a ready future resolving to `None` so the caller can use `return -/// reject_event(...)` directly inside a `filter_map` closure. -fn reject_event(reason: impl fmt::Display) -> futures::future::Ready> { +/// Marks `finalizers` as [`EventStatus::Rejected`] so that upstream sources are +/// notified of the failure, then emits [`SinkRequestBuildError`] and +/// [`ComponentEventsDropped`]. Returns a ready future resolving to `None` so +/// the caller can use `return reject_event(...)` directly inside a `filter_map` +/// closure. +fn reject_event( + finalizers: EventFinalizers, + reason: impl fmt::Display, +) -> futures::future::Ready> { let reason = reason.to_string(); + finalizers.update_status(EventStatus::Rejected); emit!(SinkRequestBuildError { error: &reason }); emit!(ComponentEventsDropped:: { count: 1, diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 3f450e95fff67..68adccedbad26 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -97,15 +97,19 @@ pub struct OpenTelemetryConfig { pub uri: Template, #[configurable(derived)] - #[configurable(metadata(docs::warnings = "The `grpc` protocol only supports `none` and `gzip`. Specifying any other algorithm causes Vector to fail at startup."))] + #[configurable(metadata( + docs::warnings = "The `grpc` protocol only supports `none` and `gzip`. Specifying any other algorithm causes Vector to fail at startup." + ))] #[serde(default)] pub compression: Compression, #[configurable(derived)] - #[configurable(metadata(docs::description = "Outbound request settings for retry, concurrency, timeout, and headers. \ + #[configurable(metadata( + docs::description = "Outbound request settings for retry, concurrency, timeout, and headers. \ For the `grpc` protocol, `request.headers` entries are forwarded as gRPC metadata — use them \ for authentication (e.g. `authorization: \"Bearer \"`) since the HTTP-only `auth` field \ - is not available for gRPC."))] + is not available for gRPC." + ))] #[serde(default)] pub request: RequestConfig, @@ -166,7 +170,8 @@ impl SinkConfig for OpenTelemetryConfig { Compression::Gzip(_) => GrpcCompression::Gzip, other => return Err(format!( "gRPC transport only supports 'none' or 'gzip' compression, got '{other}'" - ).into()), + ) + .into()), }; let config = GrpcSinkConfig { uri: self.uri.clone(), diff --git a/src/sinks/util/grpc.rs b/src/sinks/util/grpc.rs index 29c3e159bb54b..c2b6d7ad7a336 100644 --- a/src/sinks/util/grpc.rs +++ b/src/sinks/util/grpc.rs @@ -1,7 +1,10 @@ use std::task::{Context, Poll}; use futures::future::BoxFuture; -use http::{Uri, uri::{Authority, PathAndQuery, Scheme}}; +use http::{ + Uri, + uri::{Authority, PathAndQuery, Scheme}, +}; use hyper::client::HttpConnector; use hyper_openssl::HttpsConnector; use hyper_proxy::ProxyConnector; @@ -60,7 +63,11 @@ impl HyperGrpcService { .authority() .expect("gRPC service URI must have an authority (host:port) — supply a URI from `with_default_scheme`") .clone(); - Self { scheme, authority, client } + Self { + scheme, + authority, + client, + } } } diff --git a/src/sinks/vector/config.rs b/src/sinks/vector/config.rs index b182a6a2d481f..17c2cde4fb751 100644 --- a/src/sinks/vector/config.rs +++ b/src/sinks/vector/config.rs @@ -110,7 +110,8 @@ fn default_config(address: &str) -> VectorConfig { impl SinkConfig for VectorConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSinkType, Healthcheck)> { let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), false)?; - let uri = crate::sinks::util::grpc::with_default_scheme(self.address.parse()?, tls.is_tls())?; + let uri = + crate::sinks::util::grpc::with_default_scheme(self.address.parse()?, tls.is_tls())?; let client = new_client(&tls, cx.proxy())?; @@ -176,7 +177,6 @@ async fn healthcheck( } } - fn new_client( tls_settings: &MaybeTlsSettings, proxy_config: &ProxyConfig, diff --git a/src/sinks/vector/mod.rs b/src/sinks/vector/mod.rs index 08c1f1ae87b8c..03846f2b538e3 100644 --- a/src/sinks/vector/mod.rs +++ b/src/sinks/vector/mod.rs @@ -184,11 +184,15 @@ mod tests { #[test] fn test_with_default_scheme() { assert_eq!( - with_default_scheme("0.0.0.0".parse().unwrap(), false).unwrap().to_string(), + with_default_scheme("0.0.0.0".parse().unwrap(), false) + .unwrap() + .to_string(), "http://0.0.0.0/" ); assert_eq!( - with_default_scheme("0.0.0.0".parse().unwrap(), true).unwrap().to_string(), + with_default_scheme("0.0.0.0".parse().unwrap(), true) + .unwrap() + .to_string(), "https://0.0.0.0/" ); } From 4c1ec91f7459e647b4a295b5704b28d7efea3617 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 15:49:45 -0400 Subject: [PATCH 59/68] fix(opentelemetry sink): skip healthcheck for dynamic headers; reject URIs with path prefixes --- src/sinks/opentelemetry/grpc.rs | 51 +++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 89cee5a43012c..83c7c16f58bca 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -120,13 +120,23 @@ impl GrpcSinkConfig { let static_uri = if self.uri.is_dynamic() { None } else { - Some(with_default_scheme( + let uri = with_default_scheme( self.uri .get_ref() .parse() .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?, self.tls.is_some(), - )?) + )?; + let path = uri.path(); + if !path.is_empty() && path != "/" { + return Err(format!( + "gRPC sink URI must not include a path prefix (got {path:?}); \ + the RPC method path is set automatically. \ + Use a URI without a path, e.g. \"http://host:4317\"" + ) + .into()); + } + Some(uri) }; // For dynamic templates like `https://{{ host }}:4317` the static_uri is None, so @@ -185,13 +195,25 @@ impl GrpcSinkConfig { } let client = new_grpc_client(&tls, cx.proxy())?; - let healthcheck_uri = cx - .healthcheck - .uri - .clone() - .map(|u| with_default_scheme(u.uri, use_https)) - .transpose()? - .or(static_uri.clone()); + // Dynamic headers cannot be rendered without a live event, so the healthcheck cannot + // include them. Rather than running a check that omits required auth metadata (which + // would cause a false failure against a properly secured collector), skip the + // healthcheck entirely when dynamic header templates are present. + let healthcheck_uri = if !dynamic_grpc_header_templates.is_empty() { + warn!( + "Skipping gRPC healthcheck: dynamic (templated) headers are configured and \ + cannot be rendered at startup. Use static header values to re-enable the \ + healthcheck." + ); + None + } else { + cx.healthcheck + .uri + .clone() + .map(|u| with_default_scheme(u.uri, use_https)) + .transpose()? + .or(static_uri.clone()) + }; let healthcheck = Box::pin(grpc_healthcheck( client.clone(), healthcheck_uri, @@ -783,6 +805,17 @@ where Ok(u) => u, Err(e) => return reject_event(event.take_finalizers(), format!("invalid gRPC URI after rendering template: {e}")), }; + let path = u.path(); + if !path.is_empty() && path != "/" { + return reject_event( + event.take_finalizers(), + format!( + "rendered gRPC URI has an unsupported path prefix {path:?}; \ + the RPC method path is set automatically. \ + Use a URI without a path, e.g. \"http://host:4317\"" + ), + ); + } let uri = match u.scheme_str() { Some("https") if !use_https => { // The Hyper client was built without TLS (use_https=false), From 0d005bac8caaaf825b270e34d8150e89cc6afa4d Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 17:05:46 -0400 Subject: [PATCH 60/68] fix(opentelemetry sink): support URI path prefixes for gRPC reverse proxies --- src/sinks/opentelemetry/grpc.rs | 25 ++------------------- src/sinks/util/grpc.rs | 39 +++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 83c7c16f58bca..9df86ef850b6a 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -120,23 +120,13 @@ impl GrpcSinkConfig { let static_uri = if self.uri.is_dynamic() { None } else { - let uri = with_default_scheme( + Some(with_default_scheme( self.uri .get_ref() .parse() .map_err(|e| format!("invalid URI for gRPC sink: {e}"))?, self.tls.is_some(), - )?; - let path = uri.path(); - if !path.is_empty() && path != "/" { - return Err(format!( - "gRPC sink URI must not include a path prefix (got {path:?}); \ - the RPC method path is set automatically. \ - Use a URI without a path, e.g. \"http://host:4317\"" - ) - .into()); - } - Some(uri) + )?) }; // For dynamic templates like `https://{{ host }}:4317` the static_uri is None, so @@ -805,17 +795,6 @@ where Ok(u) => u, Err(e) => return reject_event(event.take_finalizers(), format!("invalid gRPC URI after rendering template: {e}")), }; - let path = u.path(); - if !path.is_empty() && path != "/" { - return reject_event( - event.take_finalizers(), - format!( - "rendered gRPC URI has an unsupported path prefix {path:?}; \ - the RPC method path is set automatically. \ - Use a URI without a path, e.g. \"http://host:4317\"" - ), - ); - } let uri = match u.scheme_str() { Some("https") if !use_https => { // The Hyper client was built without TLS (use_https=false), diff --git a/src/sinks/util/grpc.rs b/src/sinks/util/grpc.rs index c2b6d7ad7a336..cc631df7f57da 100644 --- a/src/sinks/util/grpc.rs +++ b/src/sinks/util/grpc.rs @@ -31,7 +31,11 @@ pub(crate) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { /// A Tower [`Service`] that routes gRPC requests through a Hyper HTTP/2 client, /// substituting the scheme and authority from a fixed base URI while preserving -/// the path/query set by tonic. +/// the path set by tonic. +/// +/// If the configured URI includes a path prefix (e.g. `https://gateway/grpc`), +/// that prefix is prepended to every tonic RPC path so that requests reach the +/// correct backend through a reverse proxy. /// /// Used by gRPC sinks that need to send requests to a specific endpoint using a /// shared Hyper client. @@ -39,6 +43,10 @@ pub(crate) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { pub struct HyperGrpcService { scheme: Scheme, authority: Authority, + /// Path prefix extracted from the configured URI (e.g. `"/grpc"`), or empty + /// string when the URI has no meaningful path. Never ends with `/` so it can + /// be concatenated directly with tonic's `/ServiceName/Method` paths. + path_prefix: String, pub client: hyper::Client>, BoxBody>, } @@ -63,9 +71,12 @@ impl HyperGrpcService { .authority() .expect("gRPC service URI must have an authority (host:port) — supply a URI from `with_default_scheme`") .clone(); + // Strip trailing slash so we can concatenate directly with tonic's leading-slash paths. + let path_prefix = uri.path().trim_end_matches('/').to_owned(); Self { scheme, authority, + path_prefix, client, } } @@ -81,17 +92,27 @@ impl Service> for HyperGrpcService { } fn call(&mut self, mut req: hyper::Request) -> Self::Future { - // scheme and authority are pre-validated at construction; path_and_query falls back - // to "/" if tonic omits it (it never does, but we avoid a panic either way). + // Tonic sets the RPC method path (e.g. `/pkg.Service/Method`). Prepend any + // configured path prefix so requests reach the right backend through a proxy. + // Falls back to "/" if tonic omits the path (it never does in practice). + let rpc_path = req + .uri() + .path_and_query() + .map(|pq| pq.as_str()) + .unwrap_or("/"); + let full_path: PathAndQuery = if self.path_prefix.is_empty() { + rpc_path + .parse() + .unwrap_or_else(|_| PathAndQuery::from_static("/")) + } else { + format!("{}{}", self.path_prefix, rpc_path) + .parse() + .unwrap_or_else(|_| PathAndQuery::from_static("/")) + }; let uri = Uri::builder() .scheme(self.scheme.clone()) .authority(self.authority.clone()) - .path_and_query( - req.uri() - .path_and_query() - .cloned() - .unwrap_or_else(|| PathAndQuery::from_static("/")), - ) + .path_and_query(full_path) .build() .expect("pre-validated scheme and authority always produce a valid URI"); From d0306a74d48ab85aafc6761998a3d649bbf82e55 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 17:43:18 -0400 Subject: [PATCH 61/68] fix(opentelemetry sink): reject dynamic URI templates with no static scheme prefix at startup --- src/sinks/opentelemetry/grpc.rs | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 9df86ef850b6a..8c59c08581167 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -130,11 +130,27 @@ impl GrpcSinkConfig { }; // For dynamic templates like `https://{{ host }}:4317` the static_uri is None, so - // we also check whether the static literal prefix of the template string starts with - // "https://". Only the part before the first `{{` is inspected so that a template - // like `{{ scheme }}://host:4317` does not falsely trigger TLS — in that case the - // scheme is not known at build time and events will be dropped at runtime if a - // rendered `https://` URI is encountered without a TLS connector. + // we inspect the static literal prefix of the template string (everything before the + // first `{{`) to determine the scheme at build time. + // + // If the scheme itself is dynamic (e.g. `{{ scheme }}://host:4317`) and no explicit + // `tls:` block is present, we cannot choose the right Hyper connector. Rather than + // silently dropping events whose rendered URI uses the wrong scheme at runtime, we + // fail fast here. + if self.uri.is_dynamic() && self.tls.is_none() { + let raw = self.uri.get_ref(); + let static_prefix_end = raw.find("{{").unwrap_or(raw.len()); + if !raw[..static_prefix_end].to_ascii_lowercase().contains("://") { + return Err( + "gRPC sink URI template must have a static scheme prefix \ + (e.g. \"http://{{ host }}:4317\" or \"https://{{ host }}:4317\"); \ + a fully dynamic scheme cannot be resolved at startup so the TLS \ + connector cannot be configured correctly. \ + Alternatively, add a `tls:` block to force HTTPS." + .into(), + ); + } + } let use_https = self.tls.is_some() || static_uri .as_ref() From 2967c26e8e6fbad19edddb0181ffc2cfd1596c67 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 17:53:52 -0400 Subject: [PATCH 62/68] fix(opentelemetry sink): always use TLS-capable connector to support mixed http/https URIs --- src/sinks/opentelemetry/grpc.rs | 79 ++++++--------------------------- 1 file changed, 13 insertions(+), 66 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 8c59c08581167..b91fcfc431787 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -129,43 +129,12 @@ impl GrpcSinkConfig { )?) }; - // For dynamic templates like `https://{{ host }}:4317` the static_uri is None, so - // we inspect the static literal prefix of the template string (everything before the - // first `{{`) to determine the scheme at build time. - // - // If the scheme itself is dynamic (e.g. `{{ scheme }}://host:4317`) and no explicit - // `tls:` block is present, we cannot choose the right Hyper connector. Rather than - // silently dropping events whose rendered URI uses the wrong scheme at runtime, we - // fail fast here. - if self.uri.is_dynamic() && self.tls.is_none() { - let raw = self.uri.get_ref(); - let static_prefix_end = raw.find("{{").unwrap_or(raw.len()); - if !raw[..static_prefix_end].to_ascii_lowercase().contains("://") { - return Err( - "gRPC sink URI template must have a static scheme prefix \ - (e.g. \"http://{{ host }}:4317\" or \"https://{{ host }}:4317\"); \ - a fully dynamic scheme cannot be resolved at startup so the TLS \ - connector cannot be configured correctly. \ - Alternatively, add a `tls:` block to force HTTPS." - .into(), - ); - } - } - let use_https = self.tls.is_some() - || static_uri - .as_ref() - .is_some_and(|u| u.scheme_str() == Some("https")) - || { - let raw = self.uri.get_ref().to_ascii_lowercase(); - let static_prefix_end = raw.find("{{").unwrap_or(raw.len()); - raw[..static_prefix_end].contains("https://") - }; - - let tls = if use_https { - MaybeTlsSettings::tls_client(self.tls.as_ref())? - } else { - MaybeTlsSettings::Raw(()) - }; + // The connector is always TLS-capable so it can handle both http:// and https:// + // URIs per-request, matching the behaviour of the HTTP sink. Whether the scheme + // defaults to http or https when the URI has no explicit scheme is controlled by + // whether a `tls:` block is configured. + let tls_configured = self.tls.is_some(); + let tls = MaybeTlsSettings::tls_client(self.tls.as_ref())?; let use_gzip = self.compression == GrpcCompression::Gzip; @@ -216,7 +185,7 @@ impl GrpcSinkConfig { cx.healthcheck .uri .clone() - .map(|u| with_default_scheme(u.uri, use_https)) + .map(|u| with_default_scheme(u.uri, tls_configured)) .transpose()? .or(static_uri.clone()) }; @@ -240,7 +209,7 @@ impl GrpcSinkConfig { service, dynamic_header_templates: dynamic_grpc_header_templates, uri_template: self.uri.clone(), - use_https, + tls_configured, }; Ok((VectorSink::from_event_streamsink(sink), healthcheck)) @@ -777,7 +746,9 @@ struct OtlpGrpcSink { batch_settings: BatcherSettings, service: S, uri_template: Template, - use_https: bool, + /// Used only to pick the default scheme (`http` vs `https`) when the rendered URI + /// has no explicit scheme. The connector itself handles both schemes. + tls_configured: bool, dynamic_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)>, } @@ -794,7 +765,7 @@ where })?; let uri_template = self.uri_template.clone(); - let use_https = self.use_https; + let tls_configured = self.tls_configured; let dynamic_header_templates = self.dynamic_header_templates.clone(); input @@ -807,34 +778,10 @@ where Ok(u) => u, Err(e) => return reject_event(event.take_finalizers(), format!("failed to parse rendered gRPC URI: {e}")), }; - let u = match with_default_scheme(parsed, use_https) { + let uri = match with_default_scheme(parsed, tls_configured) { Ok(u) => u, Err(e) => return reject_event(event.take_finalizers(), format!("invalid gRPC URI after rendering template: {e}")), }; - let uri = match u.scheme_str() { - Some("https") if !use_https => { - // The Hyper client was built without TLS (use_https=false), - // so it cannot complete a TLS handshake. Sending data to an - // https:// endpoint over a plaintext connector would either - // fail or silently transmit unencrypted. Drop the event and - // surface a clear error so the operator can add `tls:` or - // use a static `https://` scheme prefix. - return reject_event( - event.take_finalizers(), - "rendered gRPC URI uses \"https\" but the sink \ - has no TLS connector; add a `tls:` block or use \ - a static \"https://\" URI prefix so TLS is \ - enabled at startup", - ); - } - Some("http") | Some("https") => u, - other => { - return reject_event(event.take_finalizers(), format!( - "rendered gRPC URI has disallowed scheme {:?}; only \"http\" and \"https\" are permitted", - other.unwrap_or("") - )); - } - }; // Render dynamic headers. If any required header fails to render or produces a // non-ASCII value, drop the entire event rather than forwarding without it. From 488f831df36fee6c1761d4f566ee35beb63f16ac Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 18:07:15 -0400 Subject: [PATCH 63/68] fix(opentelemetry sink): lowercase header keys before gRPC metadata validation --- src/sinks/opentelemetry/grpc.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index b91fcfc431787..5fb9e95c508b3 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -152,7 +152,8 @@ impl GrpcSinkConfig { tonic::metadata::AsciiMetadataValue, )> = Vec::with_capacity(static_header_strings.len()); for (k, v) in &static_header_strings { - let key = tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) + let k_lower = k.to_lowercase(); + let key = tonic::metadata::AsciiMetadataKey::from_bytes(k_lower.as_bytes()) .map_err(|e| format!("invalid gRPC metadata key {k:?}: {e}"))?; let value = tonic::metadata::AsciiMetadataValue::try_from(v.as_str()) .map_err(|e| format!("invalid gRPC metadata value for key {k:?}: {e}"))?; @@ -164,7 +165,8 @@ impl GrpcSinkConfig { let mut dynamic_grpc_header_templates: Vec<(tonic::metadata::AsciiMetadataKey, Template)> = Vec::with_capacity(dynamic_header_templates_raw.len()); for (k, t) in dynamic_header_templates_raw { - let key = tonic::metadata::AsciiMetadataKey::from_bytes(k.as_bytes()) + let k_lower = k.to_lowercase(); + let key = tonic::metadata::AsciiMetadataKey::from_bytes(k_lower.as_bytes()) .map_err(|e| format!("invalid gRPC metadata key {k:?}: {e}"))?; dynamic_grpc_header_templates.push((key, t)); } @@ -173,21 +175,26 @@ impl GrpcSinkConfig { // Dynamic headers cannot be rendered without a live event, so the healthcheck cannot // include them. Rather than running a check that omits required auth metadata (which // would cause a false failure against a properly secured collector), skip the - // healthcheck entirely when dynamic header templates are present. - let healthcheck_uri = if !dynamic_grpc_header_templates.is_empty() { + // healthcheck entirely when dynamic header templates are present — unless the operator + // has supplied an explicit `healthcheck.uri` override, in which case we honour that URI + // with only the static headers (the override endpoint does not require the dynamic auth). + let explicit_healthcheck_uri = cx + .healthcheck + .uri + .clone() + .map(|u| with_default_scheme(u.uri, tls_configured)) + .transpose()?; + let healthcheck_uri = if !dynamic_grpc_header_templates.is_empty() + && explicit_healthcheck_uri.is_none() + { warn!( "Skipping gRPC healthcheck: dynamic (templated) headers are configured and \ - cannot be rendered at startup. Use static header values to re-enable the \ - healthcheck." + cannot be rendered at startup. Set a static `healthcheck.uri` override to \ + re-enable the healthcheck." ); None } else { - cx.healthcheck - .uri - .clone() - .map(|u| with_default_scheme(u.uri, tls_configured)) - .transpose()? - .or(static_uri.clone()) + explicit_healthcheck_uri.or(static_uri.clone()) }; let healthcheck = Box::pin(grpc_healthcheck( client.clone(), From 0c8ed049301e323b01c22798f3d9c4289cfed039 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 18:07:50 -0400 Subject: [PATCH 64/68] fix(docs): correct headers field path and fix indentation in OTLP examples --- .../en/highlights/2025-09-23-otlp-support.md | 20 +++++++++---------- .../components/sinks/opentelemetry.cue | 5 +++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/website/content/en/highlights/2025-09-23-otlp-support.md b/website/content/en/highlights/2025-09-23-otlp-support.md index 4f217490efa92..38f5a51703e1a 100644 --- a/website/content/en/highlights/2025-09-23-otlp-support.md +++ b/website/content/en/highlights/2025-09-23-otlp-support.md @@ -61,16 +61,16 @@ otel_sink: uri: http://localhost:5318/v1/logs method: post encoding: - codec: protobuf - protobuf: - desc_file: path/to/opentelemetry-proto.desc - message_type: opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest - use_json_names: true - framing: - method: 'bytes' - request: - headers: - content-type: 'application/x-protobuf' + codec: protobuf + protobuf: + desc_file: path/to/opentelemetry-proto.desc + message_type: opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest + use_json_names: true + framing: + method: 'bytes' + request: + headers: + content-type: 'application/x-protobuf' ``` The `desc` file was generated with the following command: diff --git a/website/cue/reference/components/sinks/opentelemetry.cue b/website/cue/reference/components/sinks/opentelemetry.cue index 22d586244c65b..5fcd812fbac1c 100644 --- a/website/cue/reference/components/sinks/opentelemetry.cue +++ b/website/cue/reference/components/sinks/opentelemetry.cue @@ -113,8 +113,9 @@ components: sinks: opentelemetry: { codec: json framing: method: newline_delimited - headers: - content-type: application/json + request: + headers: + content-type: application/json ``` 2. Sample OTEL collector config: From 0199ad4a6d75047ad331fbd3dab0b2f7b9107c16 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 18:16:03 -0400 Subject: [PATCH 65/68] fix(sinks): gate grpc util module behind sinks-opentelemetry and sinks-vector features --- src/sinks/util/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sinks/util/mod.rs b/src/sinks/util/mod.rs index 439e56606827f..8487511d605fb 100644 --- a/src/sinks/util/mod.rs +++ b/src/sinks/util/mod.rs @@ -1,5 +1,6 @@ pub mod adaptive_concurrency; pub mod auth; +#[cfg(any(feature = "sinks-opentelemetry", feature = "sinks-vector"))] pub mod grpc; // https://github.com/mcarton/rust-derivative/issues/112 #[allow(clippy::non_canonical_clone_impl)] From c8fdcc222c2bdd8db6bf94050bce73c61ffbe6e9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 18:18:35 -0400 Subject: [PATCH 66/68] test(opentelemetry sink): await healthcheck future in integration tests --- src/sinks/opentelemetry/integration_tests.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sinks/opentelemetry/integration_tests.rs b/src/sinks/opentelemetry/integration_tests.rs index 24c53e96b294f..2c535ef642801 100644 --- a/src/sinks/opentelemetry/integration_tests.rs +++ b/src/sinks/opentelemetry/integration_tests.rs @@ -78,7 +78,8 @@ async fn delivers_logs_via_grpc() { )) .unwrap(); - let (sink, _healthcheck) = config.build(SinkContext::default()).await.unwrap(); + let (sink, healthcheck) = config.build(SinkContext::default()).await.unwrap(); + healthcheck.await.expect("gRPC healthcheck failed"); let events = vec![otlp_log_event()]; // The gRPC sink emits EndpointBytesSent with the same "endpoint" and "protocol" tags as @@ -99,7 +100,10 @@ async fn delivers_logs_via_grpc_template_uri() { ) .unwrap(); - let (sink, _healthcheck) = config.build(SinkContext::default()).await.unwrap(); + let (sink, healthcheck) = config.build(SinkContext::default()).await.unwrap(); + // The URI is a dynamic template so there is no static healthcheck URI; the healthcheck + // skips gracefully and returns Ok(()) rather than failing. + healthcheck.await.expect("gRPC healthcheck failed unexpectedly for dynamic URI"); // The event carries `host` so the template renders to the collector address. let events = vec![otlp_log_event_with_host(host.split(':').next().unwrap())]; From e91e9d1cf7c15af705849261480d0b39a4e3830a Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 18:47:29 -0400 Subject: [PATCH 67/68] fix(sinks): validate URI has authority in with_default_scheme to prevent panic --- src/sinks/util/grpc.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/sinks/util/grpc.rs b/src/sinks/util/grpc.rs index cc631df7f57da..4133885dd3abb 100644 --- a/src/sinks/util/grpc.rs +++ b/src/sinks/util/grpc.rs @@ -11,22 +11,32 @@ use hyper_proxy::ProxyConnector; use tonic::body::BoxBody; use tower::Service; -/// Adds a default scheme to a URI that lacks one. +/// Adds a default scheme to a URI that lacks one, and validates that the URI has an authority. /// /// Returns the URI unchanged if a scheme is already present. Otherwise prepends /// `https` when `tls` is `true`, or `http` when `false`. Also sets the /// path-and-query to `/` if missing. +/// +/// Returns an error if the URI has no authority (host), since `HyperGrpcService` +/// requires one and would otherwise panic at request time. pub(crate) fn with_default_scheme(uri: Uri, tls: bool) -> crate::Result { - if uri.scheme().is_none() { + let uri = if uri.scheme().is_none() { let mut parts = uri.into_parts(); parts.scheme = Some(if tls { Scheme::HTTPS } else { Scheme::HTTP }); if parts.path_and_query.is_none() { parts.path_and_query = Some(PathAndQuery::from_static("/")); } - Ok(Uri::from_parts(parts)?) + Uri::from_parts(parts)? } else { - Ok(uri) + uri + }; + if uri.authority().is_none() { + return Err(format!( + "gRPC URI {uri:?} has no host; expected \"scheme://host:port\"" + ) + .into()); } + Ok(uri) } /// A Tower [`Service`] that routes gRPC requests through a Hyper HTTP/2 client, From 73032d0bbf840d7035e94973a7def963bf5d4c6a Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 18:55:36 -0400 Subject: [PATCH 68/68] fix(opentelemetry sink): emit accurate healthcheck skip warnings per root cause --- src/sinks/opentelemetry/grpc.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/sinks/opentelemetry/grpc.rs b/src/sinks/opentelemetry/grpc.rs index 5fb9e95c508b3..395454715d65b 100644 --- a/src/sinks/opentelemetry/grpc.rs +++ b/src/sinks/opentelemetry/grpc.rs @@ -178,14 +178,16 @@ impl GrpcSinkConfig { // healthcheck entirely when dynamic header templates are present — unless the operator // has supplied an explicit `healthcheck.uri` override, in which case we honour that URI // with only the static headers (the override endpoint does not require the dynamic auth). - let explicit_healthcheck_uri = cx + // Prefer an explicit healthcheck.uri override, then fall back to the static sink URI. + // with_default_scheme is applied once to the resolved value. + let raw_healthcheck_uri = cx .healthcheck .uri .clone() - .map(|u| with_default_scheme(u.uri, tls_configured)) - .transpose()?; + .map(|u| u.uri) + .or_else(|| static_uri.clone()); let healthcheck_uri = if !dynamic_grpc_header_templates.is_empty() - && explicit_healthcheck_uri.is_none() + && raw_healthcheck_uri.is_none() { warn!( "Skipping gRPC healthcheck: dynamic (templated) headers are configured and \ @@ -193,8 +195,16 @@ impl GrpcSinkConfig { re-enable the healthcheck." ); None + } else if raw_healthcheck_uri.is_none() && self.uri.is_dynamic() { + warn!( + "Skipping gRPC healthcheck: `uri` is a dynamic template and no static \ + `healthcheck.uri` override is set. To enable healthchecking, use a static URI." + ); + None } else { - explicit_healthcheck_uri.or(static_uri.clone()) + raw_healthcheck_uri + .map(|u| with_default_scheme(u, tls_configured)) + .transpose()? }; let healthcheck = Box::pin(grpc_healthcheck( client.clone(), @@ -245,10 +255,6 @@ async fn grpc_healthcheck( } let Some(uri) = uri else { - warn!( - "Skipping gRPC healthcheck: `uri` is a dynamic template and cannot be validated \ - at startup. To enable healthchecking, use a static URI." - ); return Ok(()); };