From 8bac1dbdcd27550444ea3b73e9dd4ef36172b2d4 Mon Sep 17 00:00:00 2001 From: Vladimir Zhuk Date: Tue, 24 Mar 2026 15:00:40 -0400 Subject: [PATCH 001/364] chore(datadog_metrics sink): switch series v2 and sketches to zstd compression (#24956) * chore(datadog_metrics sink): switch series v2 endpoint to zstd compression Co-Authored-By: Claude Sonnet 4.6 * chore(regression): switch statsd_to_datadog_metrics to ingress_throughput The egress_throughput goal measures compressed bytes received by the blackhole. Switching v2 series from zlib to zstd produces smaller compressed payloads (better compression ratio), which registers as a false regression in egress bytes/sec. ingress_throughput measures how fast Vector consumes statsd data from the generator, which is compression-agnostic and reflects actual pipeline performance. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(datadog_metrics sink): prevent division-by-zero in proptest with zero limits Start proptest ranges at 1 instead of 0 for uncompressed_limit and compressed_limit. The old validate_payload_size_limits rejected zero limits, but with_payload_limits is now infallible, so finish() can panic on division-by-zero when computing recommended_splits. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(datadog_metrics sink): switch sketches endpoint to zstd compression Sketches endpoint now uses zstd instead of zlib, matching Series v2. Only Series v1 remains on zlib. Validated against real Datadog API: 36/36 correctness checks passed, all 18 metrics match between v1 and v2. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(datadog_metrics sink): correct changelog to reflect sketches use zstd The changelog incorrectly stated that Sketches continue to use zlib, but the code routes Sketches to zstd compression. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor(datadog_metrics sink): move compression constants to vector-common Move zlib and zstd compression-bound constants from inline locals in DatadogMetricsCompression::max_compressed_size to lib/vector-common/src/constants.rs with descriptive names and doc comments linking to their specifications. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor(datadog_metrics sink): move inline use statements to module top Move use statements from inside test helper function bodies to the top of the test module, as is conventional in Rust. Co-Authored-By: Claude Opus 4.6 (1M context) * cargo fmt * fix(datadog_metrics sink): update integration test decompression for zstd The decompress_payload helper in integration_tests.rs hardcoded zlib decompression, but Series v2 now uses zstd. Auto-detect the compression format via is_zstd so tests work for both zlib and zstd payloads. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Pavlos Rontidis --- ...adog_metrics_zstd_series_v2.enhancement.md | 3 + lib/vector-common/src/constants.rs | 16 + .../statsd_to_datadog_metrics/experiment.yaml | 2 +- src/sinks/datadog/metrics/config.rs | 33 +- src/sinks/datadog/metrics/encoder.rs | 811 ++++++++++++------ .../datadog/metrics/integration_tests.rs | 13 +- src/sinks/datadog/metrics/request_builder.rs | 20 +- src/sinks/datadog/metrics/service.rs | 8 +- 8 files changed, 629 insertions(+), 277 deletions(-) create mode 100644 changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md diff --git a/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md b/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md new file mode 100644 index 0000000000000..1ecaef6b46b8c --- /dev/null +++ b/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md @@ -0,0 +1,3 @@ +The `datadog_metrics` sink now uses zstd compression when submitting metrics to the Series v2 (`/api/v2/series`) and Sketches endpoints. Series v1 continues to use zlib (deflate). + +authors: vladimir-dd diff --git a/lib/vector-common/src/constants.rs b/lib/vector-common/src/constants.rs index 1eeadca45b5dc..8b336d7937344 100644 --- a/lib/vector-common/src/constants.rs +++ b/lib/vector-common/src/constants.rs @@ -1,3 +1,19 @@ pub const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b]; pub const ZLIB_MAGIC: &[u8] = &[0x78]; pub const ZSTD_MAGIC: &[u8] = &[0x28, 0xB5, 0x2F, 0xFD]; + +/// Maximum size of a zlib stored (uncompressed) block in bytes. +/// See: +pub const ZLIB_STORED_BLOCK_SIZE: usize = 16384; + +/// Per-block overhead for zlib stored blocks: 1-byte header + 2-byte length + 2-byte ~length. +/// See: +pub const ZLIB_STORED_BLOCK_OVERHEAD: usize = 5; + +/// Zlib frame overhead: 2-byte header + 4-byte Adler-32 checksum trailer. +/// See: +pub const ZLIB_FRAME_OVERHEAD: usize = 6; + +/// Threshold below which zstd's `ZSTD_compressBound` adds extra margin (128 KiB). +/// See: (`ZSTD_compressBound`) +pub const ZSTD_SMALL_INPUT_THRESHOLD: usize = 128 << 10; diff --git a/regression/cases/statsd_to_datadog_metrics/experiment.yaml b/regression/cases/statsd_to_datadog_metrics/experiment.yaml index 7c0e069a9eddc..3650e2c1157a2 100644 --- a/regression/cases/statsd_to_datadog_metrics/experiment.yaml +++ b/regression/cases/statsd_to_datadog_metrics/experiment.yaml @@ -1,4 +1,4 @@ -optimization_goal: egress_throughput +optimization_goal: ingress_throughput target: name: vector diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index e296bd823d60c..a5fdccede6a14 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -87,11 +87,6 @@ impl DatadogMetricsEndpoint { } } - // Gets whether or not this is a series endpoint. - pub const fn is_series(self) -> bool { - matches!(self, Self::Series { .. }) - } - pub(super) const fn payload_limits(self) -> DatadogMetricsPayloadLimits { // from https://docs.datadoghq.com/api/latest/metrics/#submit-metrics let (uncompressed, compressed) = match self { @@ -112,6 +107,32 @@ impl DatadogMetricsEndpoint { compressed, } } + + /// Returns the compression scheme used for this endpoint. + pub(super) const fn compression(self) -> DatadogMetricsCompression { + match self { + Self::Series(SeriesApiVersion::V1) => DatadogMetricsCompression::Zlib, + _ => DatadogMetricsCompression::Zstd, + } + } +} + +/// Selects the compressor for a given Datadog metrics endpoint. +#[derive(Clone, Copy, Debug)] +pub(super) enum DatadogMetricsCompression { + /// zlib (deflate) — used by Series v1. + Zlib, + /// zstd — used by Series v2 and Sketches. + Zstd, +} + +impl DatadogMetricsCompression { + pub(super) const fn content_encoding(self) -> &'static str { + match self { + Self::Zstd => "zstd", + Self::Zlib => "deflate", + } + } } /// Maps Datadog metric endpoints to their actual URI. @@ -270,7 +291,7 @@ impl DatadogMetricsConfig { endpoint_configuration, self.default_namespace.clone(), self.series_api_version, - )?; + ); let protocol = self.get_protocol(dd_common); let sink = DatadogMetricsSink::new( diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 9f0b71c87ba98..a626244123861 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -16,7 +16,12 @@ use vector_lib::{ request_metadata::GroupedCountByteSize, }; -use super::config::{DatadogMetricsEndpoint, SeriesApiVersion}; +use vector_common::constants::{ + ZLIB_FRAME_OVERHEAD, ZLIB_STORED_BLOCK_OVERHEAD, ZLIB_STORED_BLOCK_SIZE, + ZSTD_SMALL_INPUT_THRESHOLD, +}; + +use super::config::{DatadogMetricsCompression, DatadogMetricsEndpoint, SeriesApiVersion}; use crate::{ common::datadog::{ DatadogMetricType, DatadogPoint, DatadogSeriesMetric, DatadogSeriesMetricMetadata, @@ -47,23 +52,6 @@ mod ddmetric_proto { include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); } -#[derive(Debug, Snafu)] -pub enum CreateError { - #[snafu(display("Invalid compressed/uncompressed payload size limits were given"))] - InvalidLimits, -} - -impl CreateError { - /// Gets the telemetry-friendly string version of this error. - /// - /// The value will be a short string with only lowercase letters and underscores. - pub const fn as_error_type(&self) -> &'static str { - match self { - Self::InvalidLimits => "invalid_payload_limits", - } - } -} - #[derive(Debug, Snafu)] pub enum EncoderError { #[snafu(display( @@ -133,6 +121,16 @@ impl FinishError { struct EncoderState { writer: Compressor, written: usize, + /// Upper bound on uncompressed bytes sitting in the compressor's internal buffer (written but + /// not yet flushed to `writer.get_ref()`). All compressors may buffer internally: zstd holds + /// up to 128 KB per block, zlib's BufWriter holds up to 4 KB. Since `get_ref().len()` only + /// reflects bytes that have been flushed through all layers, we track this bound to avoid + /// underestimating the compressed payload size. + /// + /// Increases by `n` on each write. Resets to `n` when a new compressed block is detected in + /// `writer.get_ref()` (the triggering write may straddle the block boundary, so `n` is a safe + /// upper bound on what remains buffered after the flush). + buffered_bound: usize, buf: Vec, processed: Vec, byte_size: GroupedCountByteSize, @@ -141,8 +139,9 @@ struct EncoderState { impl Default for EncoderState { fn default() -> Self { Self { - writer: get_compressor(), + writer: Compression::zlib_default().into(), written: 0, + buffered_bound: 0, buf: Vec::with_capacity(1024), processed: Vec::new(), byte_size: telemetry().create_request_count_byte_size(), @@ -164,45 +163,62 @@ pub struct DatadogMetricsEncoder { impl DatadogMetricsEncoder { /// Creates a new `DatadogMetricsEncoder` for the given endpoint. - pub fn new( - endpoint: DatadogMetricsEndpoint, - default_namespace: Option, - ) -> Result { + pub fn new(endpoint: DatadogMetricsEndpoint, default_namespace: Option) -> Self { let payload_limits = endpoint.payload_limits(); - Self::with_payload_limits( + + Self { endpoint, - default_namespace, - payload_limits.uncompressed, - payload_limits.compressed, - ) + default_namespace: default_namespace.map(Arc::from), + uncompressed_limit: payload_limits.uncompressed, + compressed_limit: payload_limits.compressed, + state: EncoderState { + writer: endpoint.compression().compressor(), + ..Default::default() + }, + log_schema: log_schema(), + origin_product_value: *ORIGIN_PRODUCT_VALUE, + } } +} +#[cfg(test)] +impl DatadogMetricsEncoder { /// Creates a new `DatadogMetricsEncoder` for the given endpoint, with specific payload limits. + /// + /// Only available in tests; production code always uses the API-defined limits via `new`. pub fn with_payload_limits( endpoint: DatadogMetricsEndpoint, default_namespace: Option, uncompressed_limit: usize, compressed_limit: usize, - ) -> Result { - let (uncompressed_limit, compressed_limit) = - validate_payload_size_limits(endpoint, uncompressed_limit, compressed_limit) - .ok_or(CreateError::InvalidLimits)?; - - Ok(Self { + ) -> Self { + Self { endpoint, default_namespace: default_namespace.map(Arc::from), uncompressed_limit, compressed_limit, - state: EncoderState::default(), + state: EncoderState { + writer: endpoint.compression().compressor(), + ..Default::default() + }, log_schema: log_schema(), origin_product_value: *ORIGIN_PRODUCT_VALUE, - }) + } + } + + /// Returns the current `buffered_bound` value for white-box testing of zstd block-flush reset. + fn buffered_bound(&self) -> usize { + self.state.buffered_bound } } impl DatadogMetricsEncoder { fn reset_state(&mut self) -> EncoderState { - mem::take(&mut self.state) + let new_state = EncoderState { + writer: self.endpoint.compression().compressor(), + ..Default::default() + }; + mem::replace(&mut self.state, new_state) } fn encode_single_metric(&mut self, metric: Metric) -> Result, EncoderError> { @@ -342,24 +358,36 @@ impl DatadogMetricsEncoder { // compressor might have internal state around checksums, etc, that can't be similarly // rolled back. // - // Our strategy is thus: simply take the encoded-but-decompressed size and see if it would - // fit within the compressed limit. In `get_endpoint_payload_size_limits`, we calculate the - // expected maximum overhead of zlib based on all input data being incompressible, which - // zlib ensures will be the worst case as it can figure out whether a compressed or - // uncompressed block would take up more space _before_ choosing which strategy to go with. + // Strategy: split the estimate into two parts: + // 1. Bytes already flushed to the output buffer (`get_ref().len()`) — exact compressed size. + // 2. Bytes still in the compressor's internal buffer plus this new metric — estimated via + // max_compressed_size(buffered_bound + n) (worst-case upper bound). // - // Thus, simply put, we've already accounted for the uncertainty by making our check here - // assume the worst case while our limits assume the worst case _overhead_. Maybe our - // numbers are technically off in the end, but `finish` catches that for us, too. - let compressed_len = self.state.writer.get_ref().len(); - let max_compressed_metric_len = n + max_compressed_overhead_len(n); - if compressed_len + max_compressed_metric_len > self.compressed_limit { + // All compressors may buffer data internally before flushing to the output: zstd buffers + // up to 128 KB per block, zlib's BufWriter holds up to 4 KB. `get_ref().len()` only + // reflects bytes that have been flushed through all layers. We track `buffered_bound` — + // an upper bound on uncompressed bytes written but not yet visible in `get_ref()` — and + // include it in the estimate for all compressor types. + let compression = self.endpoint.compression(); + let flushed_compressed = self.state.writer.get_ref().len(); + if flushed_compressed + compression.max_compressed_size(self.state.buffered_bound + n) + > self.compressed_limit + { return Ok(false); } // We should be safe to write our holding buffer to the compressor and store the metric. + // + // Update buffered_bound: if a new block appeared in the output (flushed_compressed grew), + // reset to n — the triggering write may straddle the block boundary, so n is a safe upper + // bound on what remains buffered. Otherwise accumulate. self.state.writer.write_all(&self.state.buf)?; self.state.written += n; + if self.state.writer.get_ref().len() > flushed_compressed { + self.state.buffered_bound = n; + } else { + self.state.buffered_bound += n; + } Ok(true) } @@ -383,7 +411,10 @@ impl DatadogMetricsEncoder { // Make sure we've written our header already. if self.state.written == 0 { match write_payload_header(self.endpoint, &mut self.state.writer) { - Ok(n) => self.state.written += n, + Ok(n) => { + self.state.written += n; + self.state.buffered_bound += n; + } Err(_) => return Ok(Some(metric)), } } @@ -874,75 +905,41 @@ fn generate_series_metrics( }]) } -fn get_compressor() -> Compressor { - // We use the "zlib default" compressor because it's all Datadog supports, and adding it - // generically to `Compression` would make things a little weird because of the conversion trait - // implementations that are also only none vs gzip. - Compression::zlib_default().into() -} - -const fn max_uncompressed_header_len() -> usize { - SERIES_PAYLOAD_HEADER.len() + SERIES_PAYLOAD_FOOTER.len() -} - -// Datadog ingest APIs accept zlib, which is what we're accounting for here. By default, zlib -// has a 2 byte header and 4 byte CRC trailer. [1] -// -// [1] https://www.zlib.net/zlib_tech.html -const ZLIB_HEADER_TRAILER: usize = 6; - -const fn max_compression_overhead_len(compressed_limit: usize) -> usize { - // We calculate the overhead as the zlib header/trailer plus the worst case overhead of - // compressing `compressed_limit` bytes, such that we assume all of the data we write may not be - // compressed at all. - ZLIB_HEADER_TRAILER + max_compressed_overhead_len(compressed_limit) -} - -const fn max_compressed_overhead_len(len: usize) -> usize { - // Datadog ingest APIs accept zlib, which is what we're accounting for here. - // - // Deflate, the underlying compression algorithm, has a technique to ensure that input data - // can't be encoded in such a way where it's expanded by a meaningful amount. This technique - // allows storing blocks of uncompressed data with only 5 bytes of overhead per block. - // Technically, the blocks can be up to 65KB in Deflate, but modern zlib implementations use - // block sizes of 16KB. [1][2] - // - // We calculate the overhead of compressing a given `len` bytes as the worst case of that many - // bytes being written to the compressor and being unable to be compressed at all - // - // [1] https://www.zlib.net/zlib_tech.html - // [2] https://www.bolet.org/~pornin/deflate-flush-fr.html - const STORED_BLOCK_SIZE: usize = 16384; - (1 + len.saturating_sub(ZLIB_HEADER_TRAILER) / STORED_BLOCK_SIZE) * 5 -} - -const fn validate_payload_size_limits( - endpoint: DatadogMetricsEndpoint, - uncompressed_limit: usize, - compressed_limit: usize, -) -> Option<(usize, usize)> { - if endpoint.is_series() { - // For series, we need to make sure the uncompressed limit can account for the header/footer - // we would add that wraps the encoded metrics up in the expected JSON object. This does - // imply that adding 1 to this limit would be allowed, and obviously we can't encode a - // series metric in a single byte, but this is just a simple sanity check, not an exhaustive - // search of the absolute bare minimum size. - let header_len = max_uncompressed_header_len(); - if uncompressed_limit <= header_len { - return None; +impl DatadogMetricsCompression { + fn compressor(self) -> Compressor { + match self { + Self::Zstd => Compression::zstd_default().into(), + Self::Zlib => Compression::zlib_default().into(), } } - // Get the maximum possible overhead of the compression container, based on the incoming - // _uncompressed_ limit. We use the uncompressed limit because we're calculating the maximum - // overhead in the case that, theoretically, none of the input data was compressible. This - // possibility is essentially impossible, but serves as a proper worst-case-scenario check. - let max_compression_overhead = max_compression_overhead_len(uncompressed_limit); - if compressed_limit <= max_compression_overhead { - return None; + /// Returns the worst-case compressed size of `n` uncompressed bytes. + /// + /// For zlib (deflate), the worst case occurs when data is entirely incompressible and stored in + /// uncompressed blocks (5 bytes overhead per 16 KB block, as per the DEFLATE spec). + /// + /// For zstd, this uses the same formula as `ZSTD_compressBound` from the zstd C library. + const fn max_compressed_size(self, n: usize) -> usize { + match self { + Self::Zlib => { + // Deflate stores incompressible data in uncompressed blocks, each with fixed + // overhead. We subtract the zlib frame from the block count since those bytes + // are not stored-block data. + n + (1 + n.saturating_sub(ZLIB_FRAME_OVERHEAD) / ZLIB_STORED_BLOCK_SIZE) + * ZLIB_STORED_BLOCK_OVERHEAD + } + Self::Zstd => { + // zstd_safe::compress_bound is not const, so we use the same formula it uses + // internally: srcSize + (srcSize >> 8) + small correction for inputs < 128 KB. + n + (n >> 8) + + if n < ZSTD_SMALL_INPUT_THRESHOLD { + (ZSTD_SMALL_INPUT_THRESHOLD - n) >> 11 + } else { + 0 + } + } + } } - - Some((uncompressed_limit, compressed_limit)) } fn write_payload_header( @@ -983,11 +980,8 @@ fn write_payload_footer( #[cfg(test)] mod tests { - use std::{ - io::{self, copy}, - num::NonZeroU32, - sync::Arc, - }; + use std::io::{self, Write as _}; + use std::{num::NonZeroU32, sync::Arc}; use bytes::{BufMut, Bytes, BytesMut}; use chrono::{DateTime, TimeZone, Timelike, Utc}; @@ -1010,19 +1004,30 @@ mod tests { use super::{ DatadogMetricsEncoder, EncoderError, ddmetric_proto, encode_proto_key_and_message, - encode_tags, encode_timestamp, generate_series_metrics, get_compressor, - get_sketch_payload_sketches_field_number, max_compression_overhead_len, - max_uncompressed_header_len, series_to_proto_message, sketch_to_proto_message, - validate_payload_size_limits, write_payload_footer, write_payload_header, + encode_tags, encode_timestamp, generate_series_metrics, + get_sketch_payload_sketches_field_number, series_to_proto_message, sketch_to_proto_message, + write_payload_footer, write_payload_header, }; use crate::{ common::datadog::DatadogMetricType, - sinks::datadog::metrics::{ - config::{DatadogMetricsEndpoint, SeriesApiVersion}, - encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, + sinks::{ + datadog::metrics::{ + config::{DatadogMetricsCompression, DatadogMetricsEndpoint, SeriesApiVersion}, + encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, + }, + util::{Compression, Compressor}, }, }; + const fn max_uncompressed_header_len(endpoint: DatadogMetricsEndpoint) -> usize { + match endpoint { + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => { + super::SERIES_PAYLOAD_HEADER.len() + super::SERIES_PAYLOAD_FOOTER.len() + } + _ => 0, + } + } + fn get_simple_counter() -> Metric { let value = MetricValue::Counter { value: 3.14 }; Metric::new("basic_counter", MetricKind::Incremental, value).with_timestamp(Some(ts())) @@ -1048,8 +1053,8 @@ mod tests { .with_timestamp(Some(ts())) } - fn get_compressed_empty_series_payload() -> Bytes { - let mut compressor = get_compressor(); + fn get_compressed_empty_series_v1_payload() -> Bytes { + let mut compressor = Compressor::from(Compression::zlib_default()); _ = write_payload_header( DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), @@ -1066,14 +1071,104 @@ mod tests { } fn get_compressed_empty_sketches_payload() -> Bytes { - get_compressor().finish().expect("should not fail").freeze() + Compressor::from(Compression::zstd_default()) + .finish() + .expect("should not fail") + .freeze() + } + + fn get_compressed_empty_series_v2_payload() -> Bytes { + Compressor::from(Compression::zstd_default()) + .finish() + .expect("should not fail") + .freeze() } - fn decompress_payload(payload: Bytes) -> io::Result { + fn decompress_zlib_payload(payload: Bytes) -> io::Result { let mut decompressor = ZlibDecoder::new(&payload[..]); let mut decompressed = BytesMut::new().writer(); - let result = copy(&mut decompressor, &mut decompressed); - result.map(|_| decompressed.into_inner().freeze()) + io::copy(&mut decompressor, &mut decompressed)?; + Ok(decompressed.into_inner().freeze()) + } + + fn decompress_zstd_payload(payload: Bytes) -> io::Result { + let decompressed = zstd::decode_all(&payload[..])?; + Ok(Bytes::from(decompressed)) + } + + /// Returns the number of bytes added to the compressor's output buffer after writing `n` + /// bytes of high-entropy data. Measures only the *incremental* bytes, not the frame overhead + /// that `finish()` would append (Adler-32 / empty final block for zlib, end frame for zstd). + /// + /// This mirrors how `try_compress_buffer` uses `max_compressed_size`: it checks how many + /// more compressed bytes would be produced, against the current running output length. + /// Compresses `n` bytes of high-entropy (worst-case for compression) data and returns the + /// total output size after `finish()`. + fn total_compressed_len(compression: DatadogMetricsCompression, n: usize) -> usize { + // Xorshift64 — period 2^64-1, passes BigCrush, produces statistically random bytes + // that neither zlib nor zstd can compress significantly. + let mut state = 0xdeadbeef_cafebabe_u64; + let data: Vec = (0..n) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect(); + let mut compressor = compression.compressor(); + compressor.write_all(&data).expect("write should succeed"); + compressor.finish().expect("finish should succeed").len() + } + + /// Validates that `max_compressed_size(n)` is a true upper bound on the compressed bytes + /// attributable to `n` uncompressed bytes, for both zlib and zstd. + /// + /// We measure `total_compressed_len(n) - total_compressed_len(0)` to strip the fixed frame + /// overhead (header + trailer) written regardless of input size, isolating the bytes + /// contributed by the data itself. + #[test] + fn max_compressed_size_is_upper_bound() { + // zlib stored-block boundary: 16 384 bytes; zstd block boundary: 131 072 bytes. + let test_sizes = [ + 0, 1, 100, 1_000, 16_383, 16_384, 16_385, 32_767, 32_768, 131_071, 131_072, 131_073, + 500_000, + ]; + + let zlib_frame = total_compressed_len(DatadogMetricsCompression::Zlib, 0); + let zstd_frame = total_compressed_len(DatadogMetricsCompression::Zstd, 0); + + // The formula must not overestimate by more than 1% of input + 64 bytes (a small + // constant that covers the zstd correction term for very small inputs). + let max_slack = |n: usize| n / 100 + 64; + + for &n in &test_sizes { + let actual_zlib = total_compressed_len(DatadogMetricsCompression::Zlib, n) - zlib_frame; + let max_zlib = DatadogMetricsCompression::Zlib.max_compressed_size(n); + assert!( + actual_zlib <= max_zlib, + "zlib n={n}: formula underestimates: actual={actual_zlib} > max={max_zlib}" + ); + assert!( + max_zlib - actual_zlib <= max_slack(n), + "zlib n={n}: formula overestimates: slack={} > {}", + max_zlib - actual_zlib, + max_slack(n) + ); + + let actual_zstd = total_compressed_len(DatadogMetricsCompression::Zstd, n) - zstd_frame; + let max_zstd = DatadogMetricsCompression::Zstd.max_compressed_size(n); + assert!( + actual_zstd <= max_zstd, + "zstd n={n}: formula underestimates: actual={actual_zstd} > max={max_zstd}" + ); + assert!( + max_zstd - actual_zstd <= max_slack(n), + "zstd n={n}: formula overestimates: slack={} > {}", + max_zstd - actual_zstd, + max_slack(n) + ); + } } fn ts() -> DateTime { @@ -1149,8 +1244,7 @@ mod tests { #[test] fn incorrect_metric_for_endpoint_causes_error() { // Series metrics can't go to the sketches endpoint. - let mut sketch_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut sketch_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let series_result = sketch_encoder.try_encode(get_simple_counter()); assert!(matches!( series_result.err(), @@ -1159,8 +1253,7 @@ mod tests { // And sketches can't go to the series endpoint. let mut series_v1_encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None); let sketch_result = series_v1_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1168,8 +1261,7 @@ mod tests { )); let mut series_v2_encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); let sketch_result = series_v2_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1380,8 +1472,7 @@ mod tests { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1404,8 +1495,7 @@ mod tests { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1427,8 +1517,7 @@ mod tests { fn encode_single_sketch_metric_with_default_limits() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let sketch = get_simple_sketch(); let expected = sketch.clone(); @@ -1450,8 +1539,7 @@ mod tests { fn encode_empty_sketch() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let sketch = Metric::new( "empty", MetricKind::Incremental, @@ -1511,77 +1599,6 @@ mod tests { assert_eq!(normal_buf, incremental_buf); } - #[test] - fn payload_size_limits_series() { - // Get the maximum length of the header/trailer data. - let header_len = max_uncompressed_header_len(); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - header_len, - usize::MAX, - ); - assert_eq!(result, None); - - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - header_len + 1, - usize::MAX, - ); - assert_eq!(result, Some((header_len + 1, usize::MAX))); - - // Get the maximum compressed overhead length, based on our input uncompressed size. This - // represents the worst case overhead based on the input data (of length usize::MAX, in this - // case) being entirely incompressible. - let compression_overhead_len = max_compression_overhead_len(usize::MAX); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - usize::MAX, - compression_overhead_len, - ); - assert_eq!(result, None); - - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - usize::MAX, - compression_overhead_len + 1, - ); - assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); - } - - #[test] - fn payload_size_limits_sketches() { - // There's no lower bound on uncompressed size for the sketches payload. - let result = validate_payload_size_limits(DatadogMetricsEndpoint::Sketches, 0, usize::MAX); - assert_eq!(result, Some((0, usize::MAX))); - - // Get the maximum compressed overhead length, based on our input uncompressed size. This - // represents the worst case overhead based on the input data (of length usize::MAX, in this - // case) being entirely incompressible. - let compression_overhead_len = max_compression_overhead_len(usize::MAX); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Sketches, - usize::MAX, - compression_overhead_len, - ); - assert_eq!(result, None); - - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Sketches, - usize::MAX, - compression_overhead_len + 1, - ); - assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); - } - #[test] fn default_payload_limits_are_endpoint_aware() { let v1 = DatadogMetricsEndpoint::Series(SeriesApiVersion::V1).payload_limits(); @@ -1609,8 +1626,7 @@ mod tests { let mut encoder = DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None, - ) - .expect("default payload size limits should be valid"); + ); let mut next_pending = Vec::new(); let mut hit_limit = false; @@ -1658,14 +1674,14 @@ mod tests { // We manually create the encoder with an arbitrarily low "uncompressed" limit but high // "compressed" limit to exercise the codepath that should avoid encoding a metric when the // uncompressed payload would exceed the limit. - let header_len = max_uncompressed_header_len(); + let header_len = + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)); let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None, header_len + 1, usize::MAX, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our uncompressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1684,11 +1700,11 @@ mod tests { let (payload, processed) = result.unwrap(); assert_eq!( payload.uncompressed_byte_size, - max_uncompressed_header_len() + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)) ); assert_eq!( payload.into_payload(), - get_compressed_empty_series_payload() + get_compressed_empty_series_v1_payload() ); assert_eq!(processed.len(), 0); } @@ -1703,8 +1719,7 @@ mod tests { None, 1, usize::MAX, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our uncompressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1740,8 +1755,7 @@ mod tests { None, uncompressed_limit, compressed_limit, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our compressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1760,11 +1774,11 @@ mod tests { let (payload, processed) = result.unwrap(); assert_eq!( payload.uncompressed_byte_size, - max_uncompressed_header_len() + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)) ); assert_eq!( payload.into_payload(), - get_compressed_empty_series_payload() + get_compressed_empty_series_v1_payload() ); assert_eq!(processed.len(), 0); } @@ -1775,14 +1789,13 @@ mod tests { // "uncompressed" limit to exercise the codepath that should avoid encoding a metric when the // compressed payload would exceed the limit. let uncompressed_limit = 128; - let compressed_limit = 16; + let compressed_limit = 32; let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Sketches, None, uncompressed_limit, compressed_limit, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our compressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1807,6 +1820,284 @@ mod tests { assert_eq!(processed.len(), 0); } + #[test] + fn encode_series_v2_breaks_out_when_limit_reached_compressed() { + // We manually create the encoder with an arbitrarily low "compressed" limit but high + // "uncompressed" limit to exercise the codepath that should avoid encoding a metric when the + // compressed payload would exceed the limit. + let uncompressed_limit = 128; + let compressed_limit = 32; + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + uncompressed_limit, + compressed_limit, + ); + + // Trying to encode a metric that would cause us to exceed our compressed limits will + // _not_ return an error from `try_encode`, but instead will simply return back the metric + // as it could not be added. + let counter = get_simple_counter(); + let result = encoder.try_encode(counter.clone()); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Some(counter)); + + // And similarly, since we didn't actually encode a metric, we _should_ be able to finish + // this payload, but it will be empty (effectively, the header/footer will exist) and no + // processed metrics should be returned. + let result = encoder.finish(); + assert!(result.is_ok()); + + let (payload, processed) = result.unwrap(); + assert_eq!(payload.uncompressed_byte_size, 0); + assert_eq!( + payload.into_payload(), + get_compressed_empty_series_v2_payload() + ); + assert_eq!(processed.len(), 0); + } + + #[test] + fn zstd_v2_payload_never_exceeds_512kb_with_incompressible_data() { + // End-to-end regression test using the real 512 KB compressed limit. + // + // Metric names are generated with a xorshift64 PRNG producing random printable ASCII + // (6.5 bits of entropy per byte), making them effectively incompressible for zstd. + // This makes the capacity estimate tight, so the test validates both directions: + // + // Safety (upper bound): payload ≤ 512 KB. + // Without the fix, the encoder ignores zstd's internal 128 KB buffer. When the + // encoder finally stops, finish() flushes that hidden buffer on top of the already + // ~511 KB payload → ~639 KB → TooLarge. + // + // Utilization (lower bound): payload > 95% of 512 KB. + // With incompressible data, actual_compressed ≈ max_cs(uncompressed), so the + // estimate is tight. The ~2.5% gap comes from: (1) compressible proto framing + // (field tags, timestamps, metadata) that zstd compresses better than max_cs + // predicts, (2) the max_cs overhead term (~0.4%), and (3) one-metric stopping + // granularity (~1%). + + // xorshift64 PRNG: 5000 random printable ASCII chars per metric name (0x21..0x7E, + // 93 values ≈ 6.5 bits/byte). Long names ensure the random portion dominates the + // compressible proto framing, maximizing utilization. + const PRINTABLE_START: u8 = 0x21; + const PRINTABLE_END: u8 = 0x7E; + const PRINTABLE_LEN: u64 = (PRINTABLE_END - PRINTABLE_START + 1) as u64; // 93 + let mut xor_state = 0xdeadbeef_cafebabe_u64; + let mut next_name = || -> String { + std::iter::once('m') + .chain((0..4999).map(|_| { + xor_state ^= xor_state << 13; + xor_state ^= xor_state >> 7; + xor_state ^= xor_state << 17; + (PRINTABLE_START + (xor_state % PRINTABLE_LEN) as u8) as char + })) + .collect() + }; + + let mut encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); + + let mut accepted = 0usize; + loop { + let metric = Metric::new( + next_name(), + MetricKind::Incremental, + MetricValue::Counter { + value: (accepted + 1) as f64, + }, + ) + .with_timestamp(Some(ts())); + + match encoder.try_encode(metric) { + Ok(None) => accepted += 1, + Ok(Some(_)) => break, + Err(e) => panic!("unexpected encoding error: {e}"), + } + } + + assert!(accepted > 0, "at least one metric must be accepted"); + + let compressed_limit = DatadogMetricsEndpoint::Series(SeriesApiVersion::V2) + .payload_limits() + .compressed; + + let (payload, _) = encoder.finish().unwrap_or_else(|e| { + panic!( + "finish() returned an error after {accepted} metrics — \ + the capacity estimate failed to prevent overflow: {e:?}" + ) + }); + let payload_len = payload.into_payload().len(); + + // Safety: the hard limit must never be exceeded. + assert!( + payload_len <= compressed_limit, + "payload ({payload_len} bytes) exceeded the {compressed_limit}-byte compressed limit" + ); + + // Utilization: the encoder should use at least 95% of the available capacity. + let min_utilization = compressed_limit * 95 / 100; + assert!( + payload_len > min_utilization, + "payload ({payload_len} bytes) is below 95% of the {compressed_limit}-byte limit \ + ({min_utilization} bytes) — estimate may be over-conservative" + ); + } + + #[test] + fn compressed_limit_is_respected_regardless_of_compressor_internal_buffering() { + // Regression test for zstd's internal buffering hiding the compressed-size check. + // + // zstd buffers up to 128 KB internally before flushing a block to the output Vec. + // The old capacity check used `get_ref().len()` alone as "compressed bytes so far", which + // returns 0 until zstd's first 128 KB block completes. This caused the encoder to accept + // metrics indefinitely — finish() would then return TooLarge, triggering expensive + // re-encoding. + // + // The fix splits the estimate: exact compressed size for flushed blocks, plus + // max_compressed_size for the unflushed portion (bytes still in zstd's internal buffer). + // This is accurate for flushed data and bounded for unflushed data. + // + // At compressed_limit=512, no zstd block will flush (needs 128 KB of input), so + // get_ref().len() stays 0 throughout. The old code would accept all 100 metrics; + // the new code stops after a handful. + let compressed_limit = 512; + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + 1_000_000, + compressed_limit, + ); + + let mut accepted = 0; + for i in 0..100 { + let metric = Metric::new( + format!("counter_{i:0>20}"), + MetricKind::Incremental, + MetricValue::Counter { + value: (i + 1) as f64, + }, + ) + .with_timestamp(Some(ts())); + match encoder.try_encode(metric) { + Ok(None) => accepted += 1, + Ok(Some(_)) => break, + Err(e) => panic!("unexpected encoding error: {e}"), + } + } + + assert!(accepted > 0, "encoder should accept at least one metric"); + assert!( + accepted < 10, + "encoder accepted too many metrics — compressed limit was likely not enforced (accepted={accepted})" + ); + + let result = encoder.finish(); + assert!( + result.is_ok(), + "finish() must not return TooLarge: {:?}", + result.err() + ); + let (payload, _) = result.unwrap(); + assert!( + payload.into_payload().len() <= compressed_limit, + "payload exceeded compressed_limit" + ); + } + + #[test] + fn zstd_buffered_bound_resets_to_last_metric_size_after_block_flush() { + // White-box test: directly verifies that buffered_bound resets to exactly n (the last + // metric's encoded size) when a zstd block flush occurs, not to 0 or some other value. + // + // buffered_bound is an upper bound on bytes in zstd's internal buffer. On each write it + // accumulates (+= n). When a flush is detected (get_ref().len() grows), it resets to n — + // meaning only the triggering write could straddle the block boundary. + // + // If it reset to 0 instead, subsequent capacity checks would degenerate to + // flushed_compressed + max_cs(n) + // which vastly underestimates for any data written after the flush, re-introducing the + // original blind spot. If it failed to reset at all, the encoder would become + // over-conservative and reject valid metrics after the first flush. + // + // Strategy: + // 1. Measure a single metric's encoded size by inspecting buffered_bound after one write. + // 2. Feed metrics into a second encoder (with unlimited limits) until buffered_bound + // *decreases*, which signals a block flush. Assert the post-flush value equals + // exactly one metric's encoded size. + + let make_metric = |i: usize| { + // Fixed-width name (600-char zero-padded) gives a constant per-metric encoded size. + // Value is (i + 1) to ensure it is never 0.0: proto3 omits default (zero) values, + // which would make the first metric smaller than the rest. + Metric::new( + format!("counter_{i:0>600}"), + MetricKind::Incremental, + MetricValue::Counter { + value: (i + 1) as f64, + }, + ) + .with_timestamp(Some(ts())) + }; + + // Step 1: measure a single metric's encoded size. + let metric_size = { + let mut probe = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + usize::MAX, + usize::MAX, + ); + assert!( + probe.try_encode(make_metric(0)).unwrap().is_none(), + "first metric must be accepted" + ); + probe.buffered_bound() + }; + assert!(metric_size > 0, "encoded metric must be non-empty"); + + // Step 2: encode metrics until buffered_bound decreases (= flush detected). + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + usize::MAX, + usize::MAX, + ); + + let mut prev_buffered = 0usize; + let mut flush_seen = false; + + for i in 0..1000 { + let metric = make_metric(i); + match encoder.try_encode(metric) { + Ok(None) => {} + Ok(Some(_)) => panic!("unexpected rejection at i={i} with unlimited limits"), + Err(e) => panic!("unexpected error at i={i}: {e}"), + } + + let curr = encoder.buffered_bound(); + + if curr < prev_buffered { + // A block flush just occurred: buffered_bound must reset to exactly n. + assert_eq!( + curr, metric_size, + "after block flush, buffered_bound should reset to one metric's encoded size \ + ({metric_size} bytes) but got {curr}" + ); + flush_seen = true; + break; + } + + prev_buffered = curr; + } + + assert!( + flush_seen, + "no zstd block flush detected after 1000 metrics — increase loop bound or metric size" + ); + } + fn arb_counter_metric() -> impl Strategy { let name = string_regex("[a-zA-Z][a-zA-Z0-9_]{8,96}").expect("regex should not be invalid"); let value = ARB_POSITIVE_F64; @@ -1827,9 +2118,9 @@ mod tests { proptest! { #[test] - fn encoding_check_for_payload_limit_edge_cases( - uncompressed_limit in 0..64_000_000usize, - compressed_limit in 0..10_000_000usize, + fn encoding_check_for_payload_limit_edge_cases_v1( + uncompressed_limit in 1..64_000_000usize, + compressed_limit in 1..10_000_000usize, metric in arb_counter_metric(), ) { // We simply try to encode a single metric into an encoder, and make sure that when we @@ -1838,25 +2129,51 @@ mod tests { // // We check this with targeted unit tests as well but this is some cheap insurance to // show that we're hopefully not missing any particular corner cases. - let result = DatadogMetricsEncoder::with_payload_limits( + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + None, + uncompressed_limit, + compressed_limit, + ); + _ = encoder.try_encode(metric); + + if let Ok((payload, _processed)) = encoder.finish() { + let payload = payload.into_payload(); + prop_assert!(payload.len() <= compressed_limit); + + // V1 uses zlib/deflate. + let result = decompress_zlib_payload(payload); + prop_assert!(result.is_ok()); + + let decompressed = result.unwrap(); + prop_assert!(decompressed.len() <= uncompressed_limit); + } + } + + #[test] + fn encoding_check_for_payload_limit_edge_cases_v2( + uncompressed_limit in 1..10_000_000usize, + compressed_limit in 1..1_000_000usize, + metric in arb_counter_metric(), + ) { + let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None, uncompressed_limit, compressed_limit, ); - if let Ok(mut encoder) = result { - _ = encoder.try_encode(metric); + _ = encoder.try_encode(metric); - if let Ok((payload, _processed)) = encoder.finish() { - let payload = payload.into_payload(); - prop_assert!(payload.len() <= compressed_limit); + if let Ok((payload, _processed)) = encoder.finish() { + let payload = payload.into_payload(); + prop_assert!(payload.len() <= compressed_limit); - let result = decompress_payload(payload); - prop_assert!(result.is_ok()); + // V2 uses zstd. + let result = decompress_zstd_payload(payload); + prop_assert!(result.is_ok()); - let decompressed = result.unwrap(); - prop_assert!(decompressed.len() <= uncompressed_limit); - } + let decompressed = result.unwrap(); + prop_assert!(decompressed.len() <= uncompressed_limit); } } } diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index e63d4c35e73c9..176c2153d88e0 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -29,6 +29,7 @@ use crate::{ DATA_VOLUME_SINK_TAGS, SINK_TAGS, assert_data_volume_sink_compliance, assert_sink_compliance, }, + compression::is_zstd, map_event_batch_stream, }, }; @@ -142,10 +143,14 @@ async fn start_test(events: Vec) -> (Vec, Receiver<(http::request: } fn decompress_payload(payload: Vec) -> std::io::Result> { - let mut decompressor = ZlibDecoder::new(&payload[..]); - let mut decompressed = Vec::new(); - let result = std::io::copy(&mut decompressor, &mut decompressed); - result.map(|_| decompressed) + if is_zstd(&payload) { + zstd::decode_all(&payload[..]) + } else { + let mut decompressor = ZlibDecoder::new(&payload[..]); + let mut decompressed = Vec::new(); + std::io::copy(&mut decompressor, &mut decompressed)?; + Ok(decompressed) + } } #[tokio::test] diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 3b1c2f4607d39..ba8a7723dbf07 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -9,19 +9,13 @@ use vector_lib::{ use super::{ config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration, SeriesApiVersion}, - encoder::{CreateError, DatadogMetricsEncoder, EncoderError, FinishError}, + encoder::{DatadogMetricsEncoder, EncoderError, FinishError}, service::DatadogMetricsRequest, }; use crate::sinks::util::{IncrementalRequestBuilder, metadata::RequestMetadataBuilder}; #[derive(Debug, Snafu)] pub enum RequestBuilderError { - #[snafu( - context(false), - display("Failed to build the request builder: {source}") - )] - FailedToBuild { source: CreateError }, - #[snafu(context(false), display("Failed to encode metric: {source}"))] FailedToEncode { source: EncoderError }, @@ -40,7 +34,6 @@ impl RequestBuilderError { /// many events were dropped as a result. pub fn into_parts(self) -> (String, &'static str, u64) { match self { - Self::FailedToBuild { source } => (source.to_string(), source.as_error_type(), 0), // Encoding errors always happen at the per-metric level, so we could only ever drop a // single metric/event at a time. Self::FailedToEncode { source } => (source.to_string(), source.as_error_type(), 1), @@ -81,18 +74,18 @@ impl DatadogMetricsRequestBuilder { endpoint_configuration: DatadogMetricsEndpointConfiguration, default_namespace: Option, series_api_version: SeriesApiVersion, - ) -> Result { - Ok(Self { + ) -> Self { + Self { endpoint_configuration, series_encoder: DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Series(series_api_version), default_namespace.clone(), - )?, + ), sketches_encoder: DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Sketches, default_namespace, - )?, - }) + ), + } } const fn get_encoder( @@ -247,6 +240,7 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< payload, uri, content_type: ddmetrics_metadata.endpoint.content_type(), + content_encoding: ddmetrics_metadata.endpoint.compression().content_encoding(), finalizers: ddmetrics_metadata.finalizers, metadata: request_metadata, } diff --git a/src/sinks/datadog/metrics/service.rs b/src/sinks/datadog/metrics/service.rs index e40c7cd91c406..024493ed88fc2 100644 --- a/src/sinks/datadog/metrics/service.rs +++ b/src/sinks/datadog/metrics/service.rs @@ -44,6 +44,7 @@ pub struct DatadogMetricsRequest { pub payload: Bytes, pub uri: Uri, pub content_type: &'static str, + pub content_encoding: &'static str, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, } @@ -63,11 +64,6 @@ impl DatadogMetricsRequest { HeaderValue::from_str(&key).expect("API key should be only valid ASCII characters") }, ); - // Requests to the metrics endpoints can be compressed, and there's almost no reason to - // _not_ compress them given tha t metric data, when encoded, is very repetitive. Thus, - // here and through the sink code, we always compress requests. Datadog also only supports - // zlib (DEFLATE) compression, which is why it's hard-coded here vs being set via the common - // `Compression` value that most sinks utilize. let request = Request::post(self.uri) .header("DD-API-KEY", api_key) // TODO: The Datadog Agent sends this header to indicate the version of the Go library @@ -82,7 +78,7 @@ impl DatadogMetricsRequest { // this header. .header("DD-Agent-Payload", "4.87.0") .header(CONTENT_TYPE, self.content_type) - .header(CONTENT_ENCODING, "deflate"); + .header(CONTENT_ENCODING, self.content_encoding); request.body(Body::from(self.payload)) } From 753cf9a7c03a5cef85a29f3561666d94e9084d8b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 24 Mar 2026 15:57:50 -0400 Subject: [PATCH 002/364] docs(internal docs): consolidate reference documentation into detailed docs table in AGENTS.md (#25036) * docs: consolidate reference documentation into detailed docs table in AGENTS.md Co-Authored-By: Claude Sonnet 4.6 * docs(internal docs): add PR title semantic convention section to AGENTS.md Co-Authored-By: Claude Sonnet 4.6 * docs(internal docs): fix markdown lint errors in PR title section Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- AGENTS.md | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 580018abf0283..e70a963047d9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -191,6 +191,12 @@ Then: `chmod +x .git/hooks/pre-push` | Topic | Document | |-------|----------| | Rust style patterns | [docs/RUST_STYLE.md](docs/RUST_STYLE.md) | +| Code style rules (formatting, const strings, organization) | [STYLE.md](STYLE.md) | +| System architecture (sources, transforms, sinks, topology) | [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | +| Component specification (naming, configuration, health checks) | [docs/specs/component.md](docs/specs/component.md) | +| Instrumentation requirements (event/metric naming) | [docs/specs/instrumentation.md](docs/specs/instrumentation.md) | +| How to document code changes | [docs/DOCUMENTING.md](docs/DOCUMENTING.md) | +| Adding changelog entries | [changelog.d/README.md](changelog.d/README.md) | ## Architecture Notes @@ -258,28 +264,15 @@ make build-licenses Before opening a PR, read [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md) and use it as the reference for the PR body structure and title. -## Reference Documentation +### PR Title Format -These documents provide context that AI agents and developers need when working on Vector code. +PR titles must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) spec and are validated by `.github/workflows/semantic.yml`. -### Essential for Code Changes +Examples: -- **[STYLE.md](STYLE.md)** - Code style rules (formatting, const strings, code organization) -- **[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)** - System architecture (sources, transforms, sinks, topology) -- **[docs/DEVELOPING.md](docs/DEVELOPING.md)** - Development workflow and testing - -### Component Development - -- **[docs/specs/component.md](docs/specs/component.md)** - Component specification (naming, configuration, health checks) -- **[docs/specs/instrumentation.md](docs/specs/instrumentation.md)** - Instrumentation requirements (event/metric naming) -- **[src/internal_events](src/internal_events)** - Internal event examples for telemetry - -### Adding Documentation - -- **[docs/DOCUMENTING.md](docs/DOCUMENTING.md)** - How to document code changes -- **[changelog.d/README.md](changelog.d/README.md)** - Adding changelog entries - -### Full Guides - -- **[CONTRIBUTING.md](CONTRIBUTING.md)** - Complete contributing guide -- **[website/README.md](website/README.md)** - Website development only (separate from Rust code) +```text +feat(kafka source): add consumer group lag metric +fix(loki sink): handle empty label sets correctly +docs(internal docs): update contributing guide +chore(deps): bump tokio to X +``` From 47b5b02b9fdd13b67294df65175e9eca99c4cd8b Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 16:36:30 -0400 Subject: [PATCH 003/364] chore(ci): split deny check into optional (all) and required (licenses) (#24990) ci(deny): split deny check into optional (all) and required (licenses) Co-authored-by: Pavlos Rontidis --- .github/workflows/deny.yml | 27 +++++++++++++++++++++++---- Makefile | 4 ++++ vdev/src/commands/check/deny.rs | 13 +++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 2a9696cc3f76d..52b40d1e1d535 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -26,12 +26,14 @@ on: type: string pull_request: + merge_group: + types: [checks_requested] schedule: # Same schedule as nightly.yml - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} cancel-in-progress: true permissions: @@ -48,8 +50,6 @@ jobs: timeout-minutes: 30 if: ${{ always() && (github.event_name != 'pull_request' || needs.changes.outputs.deny == 'true') }} needs: [changes] - env: - CARGO_INCREMENTAL: 0 steps: - name: Checkout branch uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -61,5 +61,24 @@ jobs: mold: false cargo-deny: true - - name: Check cargo deny advisories/licenses + - name: Check cargo deny (all) run: make check-deny + + test-deny-licenses: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + if: ${{ always() && (github.event_name != 'pull_request' || needs.changes.outputs.deny == 'true') }} + needs: [changes] + steps: + - name: Checkout branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} + + - uses: ./.github/actions/setup + with: + mold: false + cargo-deny: true + + - name: Check cargo deny licenses + run: make check-deny-licenses diff --git a/Makefile b/Makefile index c6e5d2423820c..71a324a5a7c5b 100644 --- a/Makefile +++ b/Makefile @@ -518,6 +518,10 @@ check-scripts: ## Check that scripts do not have common mistakes check-deny: ## Check advisories licenses and sources for crate dependencies ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny +.PHONY: check-deny-licenses +check-deny-licenses: ## Check licenses for crate dependencies + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny --licenses-only + .PHONY: check-events check-events: ## Check that events satisfy patterns set in https://github.com/vectordotdev/vector/blob/master/rfcs/2020-03-17-2064-event-driven-observability.md ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check events diff --git a/vdev/src/commands/check/deny.rs b/vdev/src/commands/check/deny.rs index 11d04c6710fa9..d24b7454b51da 100644 --- a/vdev/src/commands/check/deny.rs +++ b/vdev/src/commands/check/deny.rs @@ -5,10 +5,19 @@ use crate::app; /// Check for advisories, licenses, and sources for crate dependencies #[derive(clap::Args, Debug)] #[command()] -pub struct Cli {} +pub struct Cli { + /// Only check licenses + #[arg(long)] + licenses_only: bool, +} impl Cli { pub fn exec(self) -> Result<()> { + let check = if self.licenses_only { + "licenses" + } else { + "all" + }; app::exec( "cargo", [ @@ -17,7 +26,7 @@ impl Cli { "error", "--all-features", "check", - "all", + check, ], true, ) From ca9b29bc52b7c7ba857c337b25278edbad1c5e66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:17:07 -0400 Subject: [PATCH 004/364] chore(deps): bump serial_test from 3.2.0 to 3.4.0 (#25022) Bumps [serial_test](https://github.com/palfrey/serial_test) from 3.2.0 to 3.4.0. - [Release notes](https://github.com/palfrey/serial_test/releases) - [Commits](https://github.com/palfrey/serial_test/compare/v3.2.0...v3.4.0) --- updated-dependencies: - dependency-name: serial_test dependency-version: 3.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 11 ++++++----- Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d23731697fa21..229f42dd631ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9923,11 +9923,12 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" dependencies = [ - "futures", + "futures-executor", + "futures-util", "log", "once_cell", "parking_lot", @@ -9937,9 +9938,9 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.44", diff --git a/Cargo.toml b/Cargo.toml index 0a577cde861d9..83da1ac8eb616 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -214,7 +214,7 @@ vector-vrl-category = { path = "lib/vector-vrl/category" } vector-vrl-functions = { path = "lib/vector-vrl/functions", default-features = false } vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", default-features = false, features = ["arbitrary", "cli", "test", "test_framework", "stdlib-base"] } mock_instant = { version = "0.6" } -serial_test = { version = "3.2" } +serial_test = { version = "3.4" } strum = { version = "0.28", features = ["derive"] } [dependencies] From a2918e6ac4b874dc7c67b558a1e2bc9fd2b4ae81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:58:38 -0400 Subject: [PATCH 005/364] chore(deps): bump quick-junit from 0.5.2 to 0.6.0 (#25032) * chore(deps): bump quick-junit from 0.5.2 to 0.6.0 Bumps [quick-junit](https://github.com/nextest-rs/quick-junit) from 0.5.2 to 0.6.0. - [Release notes](https://github.com/nextest-rs/quick-junit/releases) - [Changelog](https://github.com/nextest-rs/quick-junit/blob/main/CHANGELOG.md) - [Commits](https://github.com/nextest-rs/quick-junit/compare/quick-junit-0.5.2...quick-junit-0.6.0) --- updated-dependencies: - dependency-name: quick-junit dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * fix(deps): update quick-junit version requirement to 0.6.0 in Cargo.toml Dependabot only bumped the version in Cargo.lock but left the version requirement in Cargo.toml at "0.5.1", which resolves to >=0.5.1,<0.6.0 and causes cargo to revert the lock file back to 0.5.2 on any build. Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Sonnet 4.6 (1M context) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 229f42dd631ed..c5efc9802ce97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8395,9 +8395,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-junit" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee9342d671fae8d66b3ae9fd7a9714dfd089c04d2a8b1ec0436ef77aee15e5f" +checksum = "e3e64c58c4c88fc1045e8fe98a1b7cec3643187e3dd678f9bbcdd8f12a6933d6" dependencies = [ "chrono", "indexmap 2.12.0", diff --git a/Cargo.toml b/Cargo.toml index 83da1ac8eb616..06b83efedb33e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -408,7 +408,7 @@ ordered-float.workspace = true percent-encoding = { version = "2.3.1", default-features = false } postgres-openssl = { version = "0.5.1", default-features = false, features = ["runtime"], optional = true } pulsar = { version = "6.7.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } -quick-junit = { version = "0.5.1" } +quick-junit = { version = "0.6.0" } rand.workspace = true rand_distr.workspace = true rdkafka = { version = "0.39.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } From 209b252deed214144fd2a6d4ed6c5ea3a3e08821 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 25 Mar 2026 15:11:04 -0400 Subject: [PATCH 006/364] chore(dev): enforce and fix markdownlint rules (#25038) * chore(markdown): move markdownlint config to project root, add fix-markdown target - Move .markdownlintrc from scripts/ to project root - Update vdev check markdown to use new path - Add `make fix-markdown` target for auto-fixing issues - Enable default rules and disable rules incompatible with Hugo's rendering pipeline (MD001, MD041, MD051, MD052, MD053) - Enforce emphasis style as asterisk (MD049) Co-Authored-By: Claude Sonnet 4.6 * chore(vdev): remove .github exclusion from markdown lint check Previously .github/ was excluded because it had unresolved violations. Those are now fixed, so all tracked markdown files are linted. Co-Authored-By: Claude Sonnet 4.6 * chore(markdown): enforce and fix markdownlint rules - Enable default rules; disable rules incompatible with Hugo rendering pipeline (MD001, MD024, MD025, MD041, MD051, MD052, MD053) and rules with intentional exceptions (MD012, MD013, MD014, MD033, MD034, MD049) - Add make fix-markdown target for auto-fixing issues - Remove .github exclusion from markdown lint (violations now fixed) - Fix all existing violations: table formatting (MD055/MD060), ordered list prefixes (MD029), non-descriptive link text (MD059), missing code block languages (MD040), missing alt text (MD045), list/heading spacing (MD022/MD032), list style (MD004), list indent (MD007) Co-Authored-By: Claude Sonnet 4.6 * chore(markdown): rename config to .markdownlint.jsonc for editor compatibility Rename .markdownlintrc to .markdownlint.jsonc so that markdownlint-cli2 (used by Zed, VS Code, and most modern editors) auto-detects the config. Add a comment pointing toward migrating CI from markdownlint-cli (legacy) to markdownlint-cli2 to unify tooling. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- .github/ISSUE_TEMPLATE/minor-release.md | 3 +- .github/PULL_REQUEST_TEMPLATE.md | 5 +- .github/SUPPORT.md | 14 ++-- .github/actions/spelling/README.md | 2 +- .github/actions/spelling/advice.md | 1 + .../.markdownlintrc => .markdownlint.jsonc | 19 ++++-- AGENTS.md | 2 +- CONTRIBUTING.md | 10 +-- Makefile | 4 ++ PRIVACY.md | 7 +- RELEASES.md | 18 ++--- VERSIONING.md | 14 ++-- distribution/docker/README.md | 12 ++-- docs/DEVELOPING.md | 4 +- docs/README.md | 4 +- docs/tutorials/sinks/1_basic_sink.md | 8 +-- ...6-1999-api-extensions-for-lua-transform.md | 16 ++--- .../2020-04-04-2221-kubernetes-integration.md | 54 +++++++-------- ...-05-25-2685-dev-workflow-simplification.md | 8 +-- rfcs/2020-05-25-2692-more-usable-logevents.md | 10 +-- rfcs/2020-07-28-3642-jmx_rfc.md | 4 +- rfcs/2020-10-15-3480-file-source-rework.md | 32 ++++----- rfcs/2021-02-23-6531-performance-testing.md | 4 +- ...1-03-26-6517-end-to-end-acknowledgement.md | 16 ++--- rfcs/2021-07-19-8216-multiple-pipelines.md | 4 +- rfcs/2021-08-13-8025-internal-tracing.md | 8 +-- rfcs/2021-08-29-8381-vrl-iteration-support.md | 25 +++---- ...-accept-metrics-in-datadog-agent-source.md | 20 +++--- ...021-10-29-8621-framing-and-codecs-sinks.md | 68 +++++++++---------- ...m-stats-along-traces-in-dd-agent-source.md | 2 +- rfcs/2021-11-12-9811-vrl-vm.md | 4 +- ...021-12-06-10298-kubernetes_logs-rewrite.md | 4 +- rfcs/2022-05-17-11532-chronicle-sink.md | 20 +++--- rfcs/2022-10-12-14742-flexible-metric-tags.md | 16 ++--- rfcs/2022-10-31-15056-tooling-revamp.md | 2 +- rfcs/2023-05-03-data-volume-metrics.md | 2 +- rfcs/README.md | 18 ++--- vdev/src/commands/check/markdown.rs | 14 ++-- website/README.md | 4 +- .../en/blog/adaptive-request-concurrency.md | 14 ++-- .../en/blog/highlights-february-2025.md | 4 +- website/content/en/blog/how-we-test-vector.md | 4 +- .../content/en/blog/tracking-allocations.md | 2 +- .../content/en/blog/vector-remap-language.md | 22 +++--- .../en/docs/administration/management.md | 10 +-- .../en/docs/administration/monitoring.md | 16 ++--- .../en/docs/administration/validating.md | 8 +-- .../content/en/docs/introduction/concepts.md | 4 +- .../reference/configuration/unit-tests.md | 36 +++++----- .../content/en/docs/reference/vrl/_index.md | 16 ++--- .../content/en/docs/reference/vrl/errors.md | 22 +++--- .../setup/going-to-prod/arch/aggregator.md | 2 +- .../docs/setup/going-to-prod/arch/unified.md | 2 +- .../setup/installation/manual/from-source.md | 2 +- .../installation/package-managers/helm.md | 4 +- .../installation/platforms/kubernetes.md | 4 +- .../guides/developer/config-autocompletion.md | 4 +- .../content/en/guides/developer/debugging.md | 2 +- .../2020-10-27-kubernetes-integration.md | 6 +- ...20-11-19-prometheus-remote-integrations.md | 2 +- .../2020-12-01-0-11-upgrade-guide.md | 12 ++-- .../2021-02-16-0-12-upgrade-guide.md | 8 +-- .../2021-08-24-vector-aggregator.md | 2 +- .../2021-08-25-0-16-upgrade-guide.md | 8 +-- .../2021-10-08-0-17-upgrade-guide.md | 12 ++-- .../2021-11-18-0-18-0-upgrade-guide.md | 8 +-- .../2021-12-28-0-19-0-upgrade-guide.md | 6 +- .../2022-02-08-0-20-0-upgrade-guide.md | 12 ++-- .../2022-03-22-0-21-0-upgrade-guide.md | 10 +-- .../2022-03-31-native-event-codecs.md | 6 +- .../2022-05-03-0-22-0-upgrade-guide.md | 6 +- .../2022-10-04-0-25-0-upgrade-guide.md | 4 +- .../2023-02-28-0-28-0-upgrade-guide.md | 2 +- .../2023-03-26-0-37-0-upgrade-guide.md | 2 +- .../2023-07-05-0-31-0-upgrade-guide.md | 2 +- .../2023-09-27-0-33-0-upgrade-guide.md | 8 +-- .../2023-11-07-0-34-0-upgrade-guide.md | 16 ++--- .../highlights/2023-11-07-new-linux-repos.md | 10 +-- .../2023-12-19-0-35-0-upgrade-guide.md | 12 ++-- .../2024-05-07-0-38-0-upgrade-guide.md | 2 +- .../2024-06-17-0-39-0-upgrade-guide.md | 2 +- .../2024-07-29-0-40-0-upgrade-guide.md | 4 +- 82 files changed, 415 insertions(+), 407 deletions(-) rename scripts/.markdownlintrc => .markdownlint.jsonc (53%) diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 105299559632b..7ab406601e4cb 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -37,6 +37,7 @@ cargo vdev release prepare --version "${NEW_VECTOR_VERSION}" --vrl-version "${NE ``` Automated steps include: + - Create a new release branch from master to freeze commits - `git fetch && git checkout origin/master && git checkout -b "${RELEASE_BRANCH}" && git push -u` - Create a new release preparation branch from `master` @@ -79,7 +80,7 @@ Automated steps include: - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - [ ] Merge release preparation branch into the release branch. - - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` + - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` - [ ] Tag new release - [ ] `git tag v"${NEW_VECTOR_VERSION}" -a -m v"${NEW_VECTOR_VERSION}"` diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2d239269beeda..0aa52163fafd3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,6 +9,7 @@ This should help the reviewers give feedback faster and with higher quality. --> ## Change Type + - [ ] Bug fix - [ ] New feature - [ ] Dependencies @@ -16,6 +17,7 @@ This should help the reviewers give feedback faster and with higher quality. --> - [ ] Performance ## Is this a breaking change? + - [ ] Yes - [ ] No @@ -36,6 +38,7 @@ Changes to CI, website, playground and similar are generally not considered user --> ## Notes + - Please read our [Vector contributor resources](https://github.com/vectordotdev/vector/tree/master/docs#getting-started). - Do not hesitate to use `@vectordotdev/vector` to reach out to us regarding this PR. - Some CI checks run only after we manually approve them. @@ -48,7 +51,7 @@ Changes to CI, website, playground and similar are generally not considered user - Feel free to push as many commits as you want. They will be squashed into one before merging. - For example, you can run `git merge origin master` and `git push`. - If this PR introduces changes Vector dependencies (modifies `Cargo.lock`), please - run `make build-licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details [here](https://crates.io/crates/dd-rust-license-tool). + run `make build-licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details on the [dd-rust-license-tool](https://crates.io/crates/dd-rust-license-tool). diff --git a/VERSIONING.md b/VERSIONING.md index a84b608f6ad80..9dd49bd92b68e 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -9,16 +9,16 @@ Please see the [FAQ](#faq) section for more info.** 1. [Convention](#convention) -1. [Public API](#public-api) +2. [Public API](#public-api) 1. [Areas that *are* covered](#areas-that-are-covered) 1. [Intended for *public* consumption](#intended-for-public-consumption) - 1. [Intended for *private* consumption](#intended-for-private-consumption) - 1. [Areas that are *NOT* covered](#areas-that-are-not-covered) -1. [FAQ](#faq) + 2. [Intended for *private* consumption](#intended-for-private-consumption) + 2. [Areas that are *NOT* covered](#areas-that-are-not-covered) +3. [FAQ](#faq) 1. [How often is Vector released?](#how-often-is-vector-released) - 1. [How does Vector treat patch and minor versions?](#how-does-vector-treat-patch-and-minor-versions) - 1. [How does Vector treat major versions \(breaking changes\)?](#how-does-vector-treat-major-versions-breaking-changes) - 1. [How does Vector treat pre-1.0 versions?](#how-does-vector-treat-pre-10-versions) + 2. [How does Vector treat patch and minor versions?](#how-does-vector-treat-patch-and-minor-versions) + 3. [How does Vector treat major versions \(breaking changes\)?](#how-does-vector-treat-major-versions-breaking-changes) + 4. [How does Vector treat pre-1.0 versions?](#how-does-vector-treat-pre-10-versions) diff --git a/distribution/docker/README.md b/distribution/docker/README.md index b6e6a66ea65e5..71cef83c6a4e8 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -105,12 +105,12 @@ Vector maintains special tags that are automatically updated whenever Vector is | Version | URL | |:-----------------|:---------------------------------------------------------| -| Latest major | `timberio/vector:latest-alpine` | -| Latest minor | `timberio/vector:.X-alpine` | -| Latest patch | `timberio/vector:.X-alpine` | -| Specific version | `timberio/vector:-alpine` | -| Latest nightly | `timberio/vector:nightly-alpine` | -| Specific nightly | `timberio/vector:nightly--alpine` | +| Latest major | `timberio/vector:latest-alpine` | +| Latest minor | `timberio/vector:.X-alpine` | +| Latest patch | `timberio/vector:.X-alpine` | +| Specific version | `timberio/vector:-alpine` | +| Latest nightly | `timberio/vector:nightly-alpine` | +| Specific nightly | `timberio/vector:nightly--alpine` | ### Source Files diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 63633d2bdac7b..9d3d50f7d6243 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -467,8 +467,8 @@ times: We use `flog` to build a sample set of log files to test sending logs from a file. This can be done with the following commands on Mac with `homebrew`. -Installation instruction for flog can be found -[here](https://github.com/mingrammer/flog#installation). +Installation instruction for flog can be found in the +[flog README](https://github.com/mingrammer/flog#installation). ```bash flog --bytes $((100 * 1024 * 1024)) > sample.log diff --git a/docs/README.md b/docs/README.md index cb3ed925a1158..443c706e38faa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ # Vector Internal Documentation -**_This folder contains internal documentation for Vector contributors. If +_**This folder contains internal documentation for Vector contributors. If you're a Vector user, please visit , which is powered by -the [website](../website) directory._** +the [website](../website) directory.**_ ## Getting started diff --git a/docs/tutorials/sinks/1_basic_sink.md b/docs/tutorials/sinks/1_basic_sink.md index a099a5530602d..c823c6a26530f 100644 --- a/docs/tutorials/sinks/1_basic_sink.md +++ b/docs/tutorials/sinks/1_basic_sink.md @@ -2,7 +2,7 @@ Let's write a basic sink for Vector. Currently, there are two styles of sink in Vector - 'event' and 'event streams'. The 'event' style sinks are deprecated, but currently a significant portion of Vector's sinks are still developed in this style. A tracking issue that covers which sinks have been converted to -'event streams' can be found [here][event_streams_tracking]. +'event streams' can be found in the [event streams tracking issue][event_streams_tracking]. This tutorial covers writing an 'event stream' Sink. @@ -300,8 +300,8 @@ Change the body of `run_inner` to look like the following: } ``` -More details about instrumenting Vector can be found -[here](https://github.com/vectordotdev/vector/blob/master/docs/specs/instrumentation.md). +More details about instrumenting Vector can be found in the +[instrumentation specification](https://github.com/vectordotdev/vector/blob/master/docs/specs/instrumentation.md). # Running our sink @@ -324,7 +324,7 @@ This simply connects a `stdin` source to our `basic` sink. ## vdev Vector provides a build tool `vdev` that simplifies the task of building Vector. Install -`vdev` using the instructions [here][vdev_install]. +`vdev` using the [installation instructions][vdev_install]. With `vdev` installed we can run Vector using: diff --git a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md index 4b8b75c032d66..abec1d261a80b 100644 --- a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md +++ b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md @@ -28,7 +28,7 @@ This RFC proposes a new API for the `lua` transform. Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lua/) has some limitations in its API. In particular, the following features are missing: -* **Nested Fields** +* **Nested Fields** Currently accessing nested fields is possible using the field path notation: @@ -44,7 +44,7 @@ Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lu See [#706](https://github.com/vectordotdev/vector/issues/706) and [#1406](https://github.com/vectordotdev/vector/issues/1406). -* **Setup Code** +* **Setup Code** Some scripts require expensive setup steps, for example, loading of modules or invoking shell commands. These steps should not be part of the main transform code. @@ -79,7 +79,7 @@ Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lu See [#1864](https://github.com/vectordotdev/vector/issues/1864). -* **Control Flow** +* **Control Flow** It should be possible to define channels for output events, similarly to how it is done in [`swimlanes`](https://vector.dev/docs/reference/transforms/swimlanes/) transform. @@ -136,7 +136,7 @@ This example is a log to metric transform which produces metric events from inco 1. There is an internal counter which is increased on each incoming log event. 2. The log events are discarded. -2. Each 10 seconds the transform produces a metric event with the count of received log events. +3. Each 10 seconds the transform produces a metric event with the count of received log events. 4. Edge cases are handled in the following way: 1. If there are no incoming invents, the metric event with the counter equal to 0 still has to be produced. 2. On Vector's shutdown the transform has to produce the final metric event with the count of received events since the last flush. @@ -645,10 +645,10 @@ The mapping between Vector data types and Lua data types is the following: | Vector Type | Lua Type | Comment | | :----------- | :-------- | :------- | -| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || -| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || -| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || -| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || +| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) | | +| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) | | +| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) | | +| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) | | | [`Timestamp`](https://vector.dev/docs/architecture/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | | [`Null`](https://vector.dev/docs/architecture/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | | [`Map`](https://vector.dev/docs/architecture/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | diff --git a/rfcs/2020-04-04-2221-kubernetes-integration.md b/rfcs/2020-04-04-2221-kubernetes-integration.md index e95ee2b4a5dbc..b2a06a68be1ad 100644 --- a/rfcs/2020-04-04-2221-kubernetes-integration.md +++ b/rfcs/2020-04-04-2221-kubernetes-integration.md @@ -135,7 +135,7 @@ it has to be deployed on every [`Node`][k8s_docs_node] in your cluster. The following diagram demonstrates how this works: - +Kubernetes deployment topology ### What We'll Accomplish @@ -149,7 +149,7 @@ The following diagram demonstrates how this works: #### Deploy using `kubectl` -1. Configure Vector: +1. Configure Vector: Before we can deploy Vector we must configure. This is done by creating a Kubernetes `ConfigMap`: @@ -173,7 +173,7 @@ The following diagram demonstrates how this works: kubectl create secret generic vector-config --from-file=vector.toml=vector.toml ``` -2. Deploy Vector! +2. Deploy Vector! Now that you have your custom `ConfigMap` ready it's time to deploy Vector. Create a `Namespace` and apply your `ConfigMap` and our recommended @@ -190,20 +190,20 @@ The following diagram demonstrates how this works: #### Deploy using Helm -1. Install [`helm`][helm_install]. +1. Install [`helm`][helm_install]. -2. Add our Helm Chart repo. +2. Add our Helm Chart repo. ```shell helm repo add vector https://charts.vector.dev helm repo update ``` -3. Configure Vector. +3. Configure Vector. TODO: address this when we decide on the helm chart internals. -4. Deploy Vector! +4. Deploy Vector! ```shell kubectl create namespace vector @@ -227,9 +227,9 @@ The following diagram demonstrates how this works: #### Deploy using Kustomize -1. Install [`kustomize`][kustomize]. +1. Install [`kustomize`][kustomize]. -1. Prepare `kustomization.yaml`. +2. Prepare `kustomization.yaml`. Use the same config as in [`kubectl` guide][anchor_tutorial_kubectl]. @@ -243,7 +243,7 @@ The following diagram demonstrates how this works: - vector-configmap.yaml ``` -1. Deploy Vector! +3. Deploy Vector! ```shell kustomize build . | kubectl apply -f - @@ -327,9 +327,7 @@ logs such that they're accessible from the following locations: - `/var/log/containers` - legacy location, kept for backward compatibility with pre `1.14` clusters. -To make our lives easier, here's a [link][k8s_src_build_container_logs_directory] -to the part of the k8s source that's responsible for building the path to the -log file. If we encounter issues, this would be a good starting point to unwrap +To make our lives easier, here's a reference to the [k8s source code responsible for building the container log path][k8s_src_build_container_logs_directory]. If we encounter issues, this would be a good starting point to unwrap the k8s code. #### Log file format @@ -603,7 +601,7 @@ This worth a separate dedicated RFC though. #### Security considerations on deployment configuration Security considerations on deployment configuration are grouped together with -other security-related measures. See [here](#deployment-hardening). +other security-related measures. See the [deployment hardening](#deployment-hardening) section. #### Other notable [`PodSpec`][k8s_api_pod_spec] properties @@ -1059,7 +1057,7 @@ We have a matrix of concerns, we'd like to ensure Vectors works properly with. - OCI (via [CRI-O](https://cri-o.io/) or [containerd](https://containerd.io/)) - [runc](https://github.com/opencontainers/runc) - [runhcs](https://github.com/Microsoft/hcsshim/tree/master/cmd/runhcs) - - see more [here][windows_in_kubernetes] + see [Windows in Kubernetes][windows_in_kubernetes] - [Kata Containers](https://github.com/kata-containers/runtime) - [gVisor](https://github.com/google/gvisor) - [Firecracker](https://github.com/firecracker-microvm/firecracker-containerd) @@ -1508,16 +1506,16 @@ nightly!) the supported Vector versions. ## Prior Art 1. [Filebeat k8s integration] -1. [Fluentbit k8s integration] -1. [Fluentd k8s integration] -1. [LogDNA k8s integration] -1. [Honeycomb integration] -1. [Bonzai logging operator] - This is approach is likely outside of the scope +2. [Fluentbit k8s integration] +3. [Fluentd k8s integration] +4. [LogDNA k8s integration] +5. [Honeycomb integration] +6. [Bonzai logging operator] - This is approach is likely outside of the scope of Vector's initial Kubernetes integration because it focuses more on deployment strategies and topologies. There are likely some very useful and interesting tactics in their approach though. -1. [Influx Helm charts] -1. [Awesome Operators List] - an "awesome list" of operators. +7. [Influx Helm charts] +8. [Awesome Operators List] - an "awesome list" of operators. ## Sales Pitch @@ -1541,7 +1539,7 @@ See [motivation](#motivation). namespaces? We'd just need to configure Vector to exclude this namespace?~~ See the [Origin filtering][anchor_origin_filtering] section. -1. ~~From what I understand, Vector requires the Kubernetes `watch` verb in order +2. ~~From what I understand, Vector requires the Kubernetes `watch` verb in order to receive updates to k8s cluster changes. This is required for the `kubernetes_pod_metadata` transform. Yet, Fluentbit [requires the `get`, `list`, and `watch` verbs][fluentbit_role]. Why don't we require the same?~~ @@ -1550,7 +1548,7 @@ See [motivation](#motivation). complete the implementation. It's really trivial to determine from a set of API calls used. See the [Deployment Hardening](#deployment-hardening) section. -1. What are some of the details that set Vector's Kubernetes integration apart? +3. What are some of the details that set Vector's Kubernetes integration apart? This is for marketing purposes and also helps us "raise the bar". ### From Mike @@ -1559,12 +1557,12 @@ See [motivation](#motivation). we want to test against? Some clusters use `docker`, some use `CRI-O`, [etc][container_runtimes]. Some even use [gVisor] or [Firecracker]. There might be differences in how different container runtimes handle logs. -1. How do we want to approach Helm Chart Repository management. -1. How do we implement liveness, readiness and startup probes? +2. How do we want to approach Helm Chart Repository management. +3. How do we implement liveness, readiness and startup probes? Readiness probe is a tricky one. See [Container probes](#container-probes). -1. Can we populate file at `terminationMessagePath` with some meaningful +4. Can we populate file at `terminationMessagePath` with some meaningful information when we exit or crash? -1. Can we allow passing arbitrary fields from the `Pod` object to the event? +5. Can we allow passing arbitrary fields from the `Pod` object to the event? Currently we only to pass `pod_id`, pod `annotations` and pod `labels`. ## Plan Of Attack diff --git a/rfcs/2020-05-25-2685-dev-workflow-simplification.md b/rfcs/2020-05-25-2685-dev-workflow-simplification.md index b5dfb5f41eed6..99a186c843f0c 100644 --- a/rfcs/2020-05-25-2685-dev-workflow-simplification.md +++ b/rfcs/2020-05-25-2685-dev-workflow-simplification.md @@ -183,7 +183,7 @@ There are tools like `hab` (from the Habitat project) and `packer` that can be u ## Outstanding Questions - Windows/Mac/FreeBSD builds via `make build` et all will produce native binaries natively, we should be review those docs. -- This RFC does not scope in integration tests beyond letting the `environment` run them. We may find motivation to explore a more **_slick_** solution in the future. +- This RFC does not scope in integration tests beyond letting the `environment` run them. We may find motivation to explore a more _**slick**_ solution in the future. ## Rationale & Alternatives @@ -207,6 +207,6 @@ Alternatives: 2. Get preliminary consensus this is good path forward 3. Add `DOCKER_SOCKET` passing and support integration testing 4. Cross-OS testing -6. Explore handling `make environment-%` commands via wildcard -7. Acceptance testing (Test including new contributor test) -8. Merge preliminary support +5. Explore handling `make environment-%` commands via wildcard +6. Acceptance testing (Test including new contributor test) +7. Merge preliminary support diff --git a/rfcs/2020-05-25-2692-more-usable-logevents.md b/rfcs/2020-05-25-2692-more-usable-logevents.md index 09ab89b35d7e5..601ef610f626a 100644 --- a/rfcs/2020-05-25-2692-more-usable-logevents.md +++ b/rfcs/2020-05-25-2692-more-usable-logevents.md @@ -183,14 +183,14 @@ In the [WASM transform](https://github.com/vectordotdev/vector/pull/2006/files) This RFC ultimately proposes the following steps: 1. Add UX improvements on `LogEvent`, particularly turning JSON into or from `LogEvent`. -1. Refactor the `PathIter` to make `vector::event::Lookup` type. -1. Add UX improvements on `Lookup` , particularly an internal `String` ↔ `Lookup` with an `Into`/`From` that does not do path parsing, as well as a `::from_str(s: String)` that does. (This also enables `"foo.bar".parse::()?`) -1. Refactor all `LogEvent` to accept `Into` values. +2. Refactor the `PathIter` to make `vector::event::Lookup` type. +3. Add UX improvements on `Lookup` , particularly an internal `String` ↔ `Lookup` with an `Into`/`From` that does not do path parsing, as well as a `::from_str(s: String)` that does. (This also enables `"foo.bar".parse::()?`) +4. Refactor all `LogEvent` to accept `Into` values. 1. Remove obsolete functionality like `insert_path` since the new `Lookup` type covers this. 2. Refactor the `keys` function to return an `Iterator` -1. Add an `Entry` style API to `LogEvent`. +5. Add an `Entry` style API to `LogEvent`. 1. Remove functionality rendered obsolete by the Entry API like `try_insert`, moving them to use the new Entry API -1. Provide `iter` and `iter_mut` functions that yield `(Lookup, Value)`. +6. Provide `iter` and `iter_mut` functions that yield `(Lookup, Value)`. 1. Remove the `all_fields` function, moving them to the new iterator. We believe these steps will provide a more ergonomic and consistent API. diff --git a/rfcs/2020-07-28-3642-jmx_rfc.md b/rfcs/2020-07-28-3642-jmx_rfc.md index b872b276574f5..7f0181b65620c 100644 --- a/rfcs/2020-07-28-3642-jmx_rfc.md +++ b/rfcs/2020-07-28-3642-jmx_rfc.md @@ -161,7 +161,7 @@ The query will return these metrics by parsing the query results and converting The type of metric for any metric marked `untyped` is unclear but likely gauges. We'll need to do deeper research to determine: 1. Whether we want to keep them. -1. What type they are if we retain them. +2. What type they are if we retain them. Naming of metrics is determined via: @@ -215,7 +215,7 @@ Additionally, as part of Vector's vision to be the "one tool" for ingesting and 1. Having users run telegraf or Prom node exporter and using Vector's Prometheus source to scrape it. We could not add the source directly to Vector and instead instruct users to run Prometheus' `jmx_exporter` and point Vector at the resulting data. -1. Or we could use something like [jmxtrans](https://github.com/jmxtrans/jmxtrans) and add a Vector OutputWriter. +2. Or we could use something like [jmxtrans](https://github.com/jmxtrans/jmxtrans) and add a Vector OutputWriter. I decided against both of these this as they would be in conflict with one of the listed principles of Vector: diff --git a/rfcs/2020-10-15-3480-file-source-rework.md b/rfcs/2020-10-15-3480-file-source-rework.md index da199d16e5c56..e7becd118c5d1 100644 --- a/rfcs/2020-10-15-3480-file-source-rework.md +++ b/rfcs/2020-10-15-3480-file-source-rework.md @@ -194,10 +194,10 @@ In the pseudocode above, read scheduling is controlled by `sort`, `should_read`, would need to answer the following questions: 1. In what order should I read the available files? -1. Should I finish one file before moving on to the next? -1. Should I back off reads to this file? -1. Should I back off reads to all files (i.e. sleep)? -1. How long should I spend working on a single file? +2. Should I finish one file before moving on to the next? +3. Should I back off reads to this file? +4. Should I back off reads to all files (i.e. sleep)? +5. How long should I spend working on a single file? These questions would then be answered by a combination of configuration, file metadata, and gathered statistics. @@ -226,9 +226,9 @@ out a magic identifier, but also the logic to update our list of watched files based on those identifiers. It needs to answer the following: 1. Given a visible path, does it contain a file I've seen before? -1. If I have seen it before, has it been renamed? -1. If I have seen it before, am I now seeing it in multiple places? -1. If I'm seeing duplicates, how should I choose which to follow? +2. If I have seen it before, has it been renamed? +3. If I have seen it before, am I now seeing it in multiple places? +4. If I'm seeing duplicates, how should I choose which to follow? This logic is mostly in one place in the current implementation, so it should not be terribly difficult to extract it. The use of `Fingerprinter` should @@ -238,9 +238,9 @@ In addition to simply consolidating the logic, we can expand and make the use of `Fingerprinter` more intelligent. We currently have three ways it can work: 1. Checksum (usually reliable but frustrating for small files) -1. Device and inode (simple and works with small files, but doesn't handle edge +2. Device and inode (simple and works with small files, but doesn't handle edge cases well) -1. First line checksum (solid for intended use case but not yet general) +3. First line checksum (solid for intended use case but not yet general) I'd first propose that we drop device and inode fingerprinting and add path-based fingerprinting in its place. This gives users the option to do the @@ -255,8 +255,8 @@ in place, the algorithm can look something like the following: 1. Read up to `max_line_length` bytes from the file starting at `ignored_header_bytes` -1. Return no fingerprint if there is no newline in the returned bytes -1. Otherwise, return the checksum of the bytes up to the first newline +2. Return no fingerprint if there is no newline in the returned bytes +3. Otherwise, return the checksum of the bytes up to the first newline This should give a good balance between usability and flexibility for the default strategy. As we implement it, we should evolve the current @@ -278,11 +278,11 @@ about providing an understandable config UI than designing the right interface. This should be driven by real world use cases. For example: 1. Ignoring existing checkpoints -1. Start at the beginning or end of existing files, optionally taking into +2. Start at the beginning or end of existing files, optionally taking into account factors like mtime -1. Start at the beginning or end of files added while we're watching (this can +3. Start at the beginning or end of files added while we're watching (this can be tricky) -1. Ordering which of the above concerns take precedence +4. Ordering which of the above concerns take precedence I would suggest a config like the following: @@ -327,8 +327,8 @@ reads, but it will require a bit of experimentation before we're able to determine if it's worthwhile. There are a few possible approaches: 1. Dispatch reads to an explicit threadpool -1. Spawn a limited number of blocking tokio tasks -1. Implement something with `iouring` +2. Spawn a limited number of blocking tokio tasks +3. Implement something with `iouring` The first two both introduce the questions of sizing and the ability of the underlying file system to enable concurrent access in a way that actually adds diff --git a/rfcs/2021-02-23-6531-performance-testing.md b/rfcs/2021-02-23-6531-performance-testing.md index e60a21b02f577..fbb5205bcb3ed 100644 --- a/rfcs/2021-02-23-6531-performance-testing.md +++ b/rfcs/2021-02-23-6531-performance-testing.md @@ -47,10 +47,10 @@ processes in our work on Vector: 1. Performance is a first-class testing concern for Vector. We will drive our process to identify regressions or opportunities for optimization as close to introduction as possible. -1. Identifying _that_ a regression has happened is often easier than _why_. We +2. Identifying _that_ a regression has happened is often easier than _why_. We will continuously improve Vector’s diagnosis tooling to reduce the time to debug and repair detected issues. -1. Performance regressions will inevitably, unintentionally make their way +3. Performance regressions will inevitably, unintentionally make their way into a release. When this happens we will treat this just like we would a correctness regression, relying on our diagnostic tools and rolling the experiences of repair back into the tooling. diff --git a/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md b/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md index b669c9ebc7c68..a69bdbc719c2f 100644 --- a/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md +++ b/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md @@ -330,39 +330,39 @@ source handles acknowledgements. The above structure provides for several considerations: -1. This the minimum amount of data that can be added to the metadata to +1. This the minimum amount of data that can be added to the metadata to fully support this feature, amounting to a single shared reference as the `Option` is optimized into the `Arc`. -2. The use of `Arc` reference counting for finalization prevents events +2. The use of `Arc` reference counting for finalization prevents events from "escaping" without providing a status indication. -3. If a source does not need or is not configured to require +3. If a source does not need or is not configured to require finalization, it will not contribute to the list of sources and so has no additional event overhead, and no additional allocations when the event is created. -4. Sending the finalization status to the source does not require any +4. Sending the finalization status to the source does not require any lookups or topology traversal. -5. No additional work is required to handle dropped sources due to +5. No additional work is required to handle dropped sources due to topology reconfiguration, other than the expected checking for a closed channel when sending. ## Drawbacks -1. This adds a base size overhead to each event, even for +1. This adds a base size overhead to each event, even for configurations that do not support or require end-to-end acknowledgement. ## Alternatives -1. The set of sources could be stored in a more customary `Vec`. This +1. The set of sources could be stored in a more customary `Vec`. This provides for merging multiple sources with a minimum of code. However, merged events already have other overhead, and it increases the data required for this array by an additional word. -2. The source could be stored as simply the unique identifier +2. The source could be stored as simply the unique identifier string. This requires that all reporting of finalization status proceed through a dictionary lookup instead of simply sending it through a channel, increasing the run-time overhead. diff --git a/rfcs/2021-07-19-8216-multiple-pipelines.md b/rfcs/2021-07-19-8216-multiple-pipelines.md index a628fe1843e08..6176d99ec2272 100644 --- a/rfcs/2021-07-19-8216-multiple-pipelines.md +++ b/rfcs/2021-07-19-8216-multiple-pipelines.md @@ -36,8 +36,8 @@ Large Vector users often require complex Vector topologies to facilitate the col This change will introduce the concept of pipelines to users. A pipeline is defined as: 1. A collection of transforms defined together, outside of the top-level configuration file -1. Able to draw input from and send output to components defined in the top-level configuration, but are isolated from other pipelines -1. Having each contained component's internal metrics tagged with the `id` of the pipeline +2. Able to draw input from and send output to components defined in the top-level configuration, but are isolated from other pipelines +3. Having each contained component's internal metrics tagged with the `id` of the pipeline Pipelines will be loaded from a `pipelines` sub-directory relative to the Vector configuration directory (e.g., `/etc/vector/pipelines`). Therefore, if a user changes the location of the Vector configuration directory they will also change the `pipelines` directory path. They are coupled. diff --git a/rfcs/2021-08-13-8025-internal-tracing.md b/rfcs/2021-08-13-8025-internal-tracing.md index 37dc48afc418b..d86a9f8ce4035 100644 --- a/rfcs/2021-08-13-8025-internal-tracing.md +++ b/rfcs/2021-08-13-8025-internal-tracing.md @@ -81,10 +81,10 @@ The step after that is a bit fuzzy, however. You could jump straight to profiling, but it has a few weaknesses: 1. It's not something that can easily be run in customer environments -1. The focus on CPU time vs wall clock time means it can miss issues like bad +2. The focus on CPU time vs wall clock time means it can miss issues like bad rate limits, waiting on downstream components, slow IO operations or syscalls, etc -1. The output tends to require interpretation by an experienced engineer, and +3. The output tends to require interpretation by an experienced engineer, and doesn't always indicate clearly where time is being spent from a Vector perspective @@ -251,10 +251,10 @@ as follows: corresponding visualizations means it would take more interpretation and external knowledge to derive the same signal. -1. Their aggregated nature would put a limit on the level of detail (e.g. no +2. Their aggregated nature would put a limit on the level of detail (e.g. no file name field on a `read` span from the file source). -1. Collecting timings directly is likely to require more explicit +3. Collecting timings directly is likely to require more explicit instrumentation code than simply adding spans. ### Do nothing diff --git a/rfcs/2021-08-29-8381-vrl-iteration-support.md b/rfcs/2021-08-29-8381-vrl-iteration-support.md index 39ab28cc9f717..020e9be84c66b 100644 --- a/rfcs/2021-08-29-8381-vrl-iteration-support.md +++ b/rfcs/2021-08-29-8381-vrl-iteration-support.md @@ -338,14 +338,14 @@ individual use-cases, this list shows one available solution per use-case. . = map_keys(., recursive: true) -> |key| { trim_start(key, "_") } ``` -11. [add prefix to all keys](https://discord.com/channels/742820443487993987/764187584452493323/883274684576182302) +1. [add prefix to all keys](https://discord.com/channels/742820443487993987/764187584452493323/883274684576182302) ```coffee . = map_keys(., recursive: true) -> |key| { "my_" + key } ``` -12. [parse message using list of Grok patterns until one matches](https://discord.com/channels/742820443487993987/764187584452493323/870353108692271104) +2. [parse message using list of Grok patterns until one matches](https://discord.com/channels/742820443487993987/764187584452493323/870353108692271104) ```coffee patterns = [] @@ -359,7 +359,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -13. [find match against list of regular expressions](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) +3. [find match against list of regular expressions](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) ```coffee matched = false @@ -377,13 +377,13 @@ individual use-cases, this list shows one available solution per use-case. matched = any(patterns) -> |pattern| { match(.message, pattern) } ``` -14. [remove prefix from keys](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) +4. [remove prefix from keys](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) ```coffee . = map_keys(. ,recursive: true) -> |key| { replace(key, "my_prefix_", "") } ``` -15. [run `encode_json` on all top-level object fields](https://discord.com/channels/742820443487993987/746070591097798688/841787442271879209) +5. [run `encode_json` on all top-level object fields](https://discord.com/channels/742820443487993987/746070591097798688/841787442271879209) ```coffee . = map_values(.) -> |value| { @@ -395,7 +395,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -16. [map key/value pairs to object with ”key” and ”value” fields](https://discord.com/channels/742820443487993987/746070591097798688/832684085771370587) +6. [map key/value pairs to object with ”key” and ”value” fields](https://discord.com/channels/742820443487993987/746070591097798688/832684085771370587) ```coffee . = { "labels": { "key1": "value1", "key2": "value2" } } @@ -405,7 +405,8 @@ individual use-cases, this list shows one available solution per use-case. } .labels = new_labels - ``` + + ```text **NOTE** this is similar to [Jq’s `to_entries` function](https://stedolan.github.io/jq/manual/#to_entries,from_entries,with_entries), @@ -422,7 +423,7 @@ individual use-cases, this list shows one available solution per use-case. . = to_entries(.) ``` -17. [run `parse_json` on multiple strings in array, and emit as multiple +1. [run `parse_json` on multiple strings in array, and emit as multiple events](https://discord.com/channels/742820443487993987/746070591097798688/832257215506415657) ```coffee @@ -432,7 +433,7 @@ individual use-cases, this list shows one available solution per use-case. ``` -18. [convert object to specific string format](https://discord.com/channels/742820443487993987/764187584452493323/824574475495407639) +2. [convert object to specific string format](https://discord.com/channels/742820443487993987/764187584452493323/824574475495407639) ```coffee . = { "key1": "value1", "key2": "value2" } @@ -451,7 +452,7 @@ individual use-cases, this list shows one available solution per use-case. "{" + join(strings, ",") + "}" ``` -19. [re-introduce previous `only_fields` functionality using iteration](https://github.com/vectordotdev/vector/issues/7347) +3. [re-introduce previous `only_fields` functionality using iteration](https://github.com/vectordotdev/vector/issues/7347) ```coffee @@ -471,7 +472,7 @@ individual use-cases, this list shows one available solution per use-case. . = filter(.) -> |key, _| { includes(only_fields, key) } ``` -20. [map complex dynamic object based on conditionals](https://github.com/vectordotdev/vector/discussions/12387#discussioncomment-2639876) +4. [map complex dynamic object based on conditionals](https://github.com/vectordotdev/vector/discussions/12387#discussioncomment-2639876) ```coffee .input = map_values(.input) -> |input| { @@ -507,7 +508,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -21. merge array of objects into single object +5. merge array of objects into single object ```coffee result = {} diff --git a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md index ac95ef626762e..c0471ed349985 100644 --- a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md +++ b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md @@ -78,7 +78,7 @@ https://vector.mycompany.tld` to forward metrics to a Vector deployment. The current `dd_url` endpoint configuration has a [conditional behavior](https://github.com/DataDog/datadog-agent/blob/main/pkg/config/config.go#L1199-L1201) (also -[here](https://github.com/DataDog/datadog-agent/blob/main/pkg/forwarder/forwarder_health.go#L131-L143)). I.e. if +[in the forwarder health check](https://github.com/DataDog/datadog-agent/blob/main/pkg/forwarder/forwarder_health.go#L131-L143)). I.e. if `dd_url` contains a known pattern (i.e. it has a suffix that matches a Datadog site) some extra hostname manipulation happens. But overall, the following paths are expected to be supported on the host behind `dd_url`: @@ -100,20 +100,20 @@ A few details about the Datadog Agents & [Datadog metrics](https://docs.datadogh [`MetricSample`](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/metric_sample.go#L81-L94) and can be of [several types](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/metric_sample.go#L20-L31) * Major Agent usecases: - * Metrics are send from corechecks (i.e. go code) - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/aggregator/sender.go#L227-L252) - * Dogstatsd metrics are converted to the `MetricSample` structure - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/dogstatsd/enrich.go#L87-L137) However Datadog Agents + * Metrics are send from corechecks (i.e. go code) via the + [aggregator sender](https://github.com/DataDog/datadog-agent/blob/main/pkg/aggregator/sender.go#L227-L252) + * Dogstatsd metrics are converted to the `MetricSample` structure in the + [dogstatsd enrich module](https://github.com/DataDog/datadog-agent/blob/main/pkg/dogstatsd/enrich.go#L87-L137). However Datadog Agents metrics are transformed before being sent, ultimately metrics accounts for two different kind of payload: * The count, gauge and rate series kind of payload, sent to `/api/v1/series` using the [JSON schema officially documented](https://docs.datadoghq.com/api/latest/metrics) with few undocumented [additional fields](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/series.go#L45-L57), but this align very well with the existing `datadog_metrics` sinks. -* The sketches kind of payload, sent to `/api/beta/sketches` and serialized as protobuf as shown - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/serializer/serializer.go#L315-L338) (it ultimately lands - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/sketch_series.go#L103-L269)). Public `.proto` - definition can be found - [here](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). +* The sketches kind of payload, sent to `/api/beta/sketches` and serialized as protobuf as shown in the + [serializer](https://github.com/DataDog/datadog-agent/blob/main/pkg/serializer/serializer.go#L315-L338) (it ultimately lands in the + [sketch_series module](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/sketch_series.go#L103-L269)). Public `.proto` + definition can be found in the + [agent-payload proto](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). Vector has a nice description of its [metrics data model](https://vector.dev/docs/architecture/data-model/metric/) and a [concise enum for diff --git a/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md b/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md index 4b2c7ace88349..87a28793d93ec 100644 --- a/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md +++ b/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md @@ -94,37 +94,37 @@ Incremental steps to execute this change. These will be converted to issues afte Overview for the current state of sinks regarding encoding: -|sink|encoding config|`.apply_rules`|notes| -|-|-|-|-| -|`aws_cloudwatch_logs`| `EncodingConfig` | ✔︎ | Enveloped in `rusoto_logs::InputLogEvent`. `Text` reads message_key() -|`aws_kinesis_firehose`| `EncodingConfig` | ✔︎ | Enveloped in `rusoto_firehose::Record` that serializes to base64. `Text` reads `message_key()` | - -|`aws_kinesis_streams`| `EncodingConfig` | ✔︎ | Enveloped in `rusoto_kinesis::PutRecordsRequestEntry`. `Text` reads `message_key()` -|`aws_s3`| `EncodingConfig` | ✔︎ | Uses `util::{RequestBuilder, Encoder, Compressor}`. `Text` reads `message_key()` -|`aws_sqs`| `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` -|`azure_blob`| `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` -|`azure_monitor_logs`| `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request -|`blackhole`| - | - | - -|`clickhouse`| `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request -|`console`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`datadog_logs`| `EncodingConfigFixed` | ✔︎ | Doesn't provide options to encode the event payload separately from the protocol -|`datadog_events`| - | ✗ | - -|`datadog_archives`| - | ✗ | Uses custom `DatadogArchivesEncoding`, which has a field `inner: StandardEncodings` which is not user-configurable -|`elasticsearch`| `EncodingConfigFixed` | ✔︎ | Reshapes event internally and implements custom `ElasticsearchEncoder` to serialize for the Elasticsearch protocol -|`file`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`gcp`| `EncodingConfig` | ✔︎ | Enveloped in HTTP request via `sinks::util::request_builder::RequestBuild`. Sets HTTP request header depending on encoding config -|`honeycomb`| - | ✗ | Embeds event as JSON under a `data` key. Enveloped in HTTP request -|`http`| `EncodingConfig` | ✔︎ | Enveloped in HTTP request. Request-level compression. Sets HTTP request header depending on encoding config -|`humio`| `EncodingConfig` | ✔︎ | Wrapper, see `splunk_hec` for more information -|`influxdb`| `EncodingConfigWithDefault` | ✔︎ | Encoding (for the protocol envelope) is done by routing event fields either into a "tags" or "fields" map that are passed into an internal function `influx_line_protocol` -|`kafka`| `EncodingConfig` | ✔︎ | Uses `Encoder` for `StandardEncodings` in `encode_input`, enveloped in `KafkaRequest` -|`logdna`| `EncodingConfigWithDefault` | ✔︎ | Builds a message by manually picking fields from the event. Enveloped in HTTP request -|`loki`| `EncodingConfig` | ✔︎ | Uses reshaping. `Text` reads `message_key()`, `Logfmt` build a key-value string. Sink has config to preprocess event by adding/removing label fields and timestamp. Enveloped in HTTP request -|`nats`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`new_relic_logs`| `EncodingConfigWithDefault` | ✔︎ | Defers to HTTP sink, uses encoding config to reshape only and convert to JSON -|`papertrail`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Serializes event using syslog and sends buffer via TCP -|`pulsar`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()`, `Avro` expects another dedicated key for the serialization schema. Serialized buffer is sent to Pulsar producer -|`redis`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Encoded message is serialized to buffer -|`sematext`| `EncodingConfigFixed` | ✔︎ | Wrapper, see `elasticsearch` for more information -|`socket`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`splunk_hec`| `EncodingConfig` | ✔︎ | Encoding is used to create a message according to the Splunk HEC protocol. There is no separate control over encoding the payload itself -|`vector`| - | - | - +| sink | encoding config | `.apply_rules` | notes | +| - | - | - | - | +| `aws_cloudwatch_logs` | `EncodingConfig` | ✔︎ | Enveloped in `rusoto_logs::InputLogEvent`. `Text` reads message_key() | +| `aws_kinesis_firehose` | `EncodingConfig` | ✔︎ | Enveloped in `rusoto_firehose::Record` that serializes to base64. `Text` reads `message_key()` | +| `aws_kinesis_streams` | `EncodingConfig` | ✔︎ | Enveloped in `rusoto_kinesis::PutRecordsRequestEntry`. `Text` reads `message_key()` | +| `aws_s3` | `EncodingConfig` | ✔︎ | Uses `util::{RequestBuilder, Encoder, Compressor}`. `Text` reads `message_key()` | +| `aws_sqs` | `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` | +| `azure_blob` | `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` | +| `azure_monitor_logs` | `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request | +| `blackhole` | - | - | - | +| `clickhouse` | `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request | +| `console` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `datadog_logs` | `EncodingConfigFixed` | ✔︎ | Doesn't provide options to encode the event payload separately from the protocol | +| `datadog_events` | - | ✗ | - | +| `datadog_archives` | - | ✗ | Uses custom `DatadogArchivesEncoding`, which has a field `inner: StandardEncodings` which is not user-configurable | +| `elasticsearch` | `EncodingConfigFixed` | ✔︎ | Reshapes event internally and implements custom `ElasticsearchEncoder` to serialize for the Elasticsearch protocol | +| `file` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `gcp` | `EncodingConfig` | ✔︎ | Enveloped in HTTP request via `sinks::util::request_builder::RequestBuild`. Sets HTTP request header depending on encoding config | +| `honeycomb` | - | ✗ | Embeds event as JSON under a `data` key. Enveloped in HTTP request | +| `http` | `EncodingConfig` | ✔︎ | Enveloped in HTTP request. Request-level compression. Sets HTTP request header depending on encoding config | +| `humio` | `EncodingConfig` | ✔︎ | Wrapper, see `splunk_hec` for more information | +| `influxdb` | `EncodingConfigWithDefault` | ✔︎ | Encoding (for the protocol envelope) is done by routing event fields either into a "tags" or "fields" map that are passed into an internal function `influx_line_protocol` | +| `kafka` | `EncodingConfig` | ✔︎ | Uses `Encoder` for `StandardEncodings` in `encode_input`, enveloped in `KafkaRequest` | +| `logdna` | `EncodingConfigWithDefault` | ✔︎ | Builds a message by manually picking fields from the event. Enveloped in HTTP request | +| `loki` | `EncodingConfig` | ✔︎ | Uses reshaping. `Text` reads `message_key()`, `Logfmt` build a key-value string. Sink has config to preprocess event by adding/removing label fields and timestamp. Enveloped in HTTP request | +| `nats` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `new_relic_logs` | `EncodingConfigWithDefault` | ✔︎ | Defers to HTTP sink, uses encoding config to reshape only and convert to JSON | +| `papertrail` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Serializes event using syslog and sends buffer via TCP | +| `pulsar` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()`, `Avro` expects another dedicated key for the serialization schema. Serialized buffer is sent to Pulsar producer | +| `redis` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Encoded message is serialized to buffer | +| `sematext` | `EncodingConfigFixed` | ✔︎ | Wrapper, see `elasticsearch` for more information | +| `socket` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `splunk_hec` | `EncodingConfig` | ✔︎ | Encoding is used to create a message according to the Splunk HEC protocol. There is no separate control over encoding the payload itself | +| `vector` | - | - | - | diff --git a/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md b/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md index 0dd290232319a..c34109a576289 100644 --- a/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md +++ b/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md @@ -100,7 +100,7 @@ found in the trace support [RFC][trace-support-pr] and may provide relevant cont ## Cross cutting concerns * Ongoing work on transforms to add `named_outputs` that is laying the ground for the same feature but on `sources`, - [one PR][named-outputs-pr] has already be merged while scheduled work is tracked [here][named-outputs-improvements]. + [one PR][named-outputs-pr] has already be merged while scheduled work is tracked in the [named outputs improvements issue][named-outputs-improvements]. * [Ongoing work on schemas][schema-rfc] will ultimately offer a programatic way of validating required fields and express constrains on incoming event for a given sink. Traces & APM stats are a good fit for that because they will be represented as standard Vector events, but sinks handling thos will expect some mandatory information. diff --git a/rfcs/2021-11-12-9811-vrl-vm.md b/rfcs/2021-11-12-9811-vrl-vm.md index 0cb103fa9cf6a..6f6450cf3903d 100644 --- a/rfcs/2021-11-12-9811-vrl-vm.md +++ b/rfcs/2021-11-12-9811-vrl-vm.md @@ -323,8 +323,8 @@ functionality. The implementation needs to occur in well defined stages to prevent dumping a massive PR that never gets over the line. -- [ ] Submit a PR with spike-level code _roughly_ demonstrating the change for - the VM. [here](https://github.com/vectordotdev/vector/pull/9829) +- [ ] Submit a [PR with spike-level code](https://github.com/vectordotdev/vector/pull/9829) _roughly_ demonstrating the change for + the VM. - [ ] Incorporate (unit and property) tests. - [ ] Document and comment the VM and the functions used to emit OpCodes. - [ ] Develop a safe API surrounding the VM (in particular function calls). diff --git a/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md b/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md index d2dccc735fd1d..f66ab9824613a 100644 --- a/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md +++ b/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md @@ -126,8 +126,8 @@ enough to not warrant a version split between the old and new code. - Replace contents of `src/kubernetes` with equivalents from `kube` 1. `src/kubernetes/client` replaced with `kube-client` - 1. `src/kubernetes/reflector` and dependencies replaced with `kube-runtime::reflector` - 1. `src/kubernetes/state` updates to minimize in-house code + 2. `src/kubernetes/reflector` and dependencies replaced with `kube-runtime::reflector` + 3. `src/kubernetes/state` updates to minimize in-house code - Ensure unit tests and integration tests show matching behavior before and after rewrite ## Future Improvements diff --git a/rfcs/2022-05-17-11532-chronicle-sink.md b/rfcs/2022-05-17-11532-chronicle-sink.md index 146008b8b9b3a..a65596fdfa3d7 100644 --- a/rfcs/2022-05-17-11532-chronicle-sink.md +++ b/rfcs/2022-05-17-11532-chronicle-sink.md @@ -128,16 +128,16 @@ The encoding of the message (text or json) can be set via an `encoding` field. ##### Schema A UDM message has the following sections. (To keep this RFC succinct, this is -not a comprehensive list, the full list can be found -[here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#udm_event_data_model).) +not a comprehensive list, the full list can be found in the +[UDM event data model reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#udm_event_data_model).) - *Metadata* Contains metadata about the event. The fields available for metadata can be - found [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#metadata). + found in the [UDM metadata field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#metadata). Mandatory fields are `event_type` and `event_timestamp`. The value of `event_type` has repercussions for which fields are mandatory - in the rest of the event. See [here](https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage#required_and_optional_fields_based_on_event_type) + in the rest of the event. See the [required and optional fields by event type](https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage#required_and_optional_fields_based_on_event_type) for the list. For example, if `event_type == "FILE_COPY"` then `src.file` becomes mandatory. **Currently it is not possible to validate a field based on the value of another field within Vector schema, so this will need to @@ -145,18 +145,18 @@ not a comprehensive list, the full list can be found - *Principal* The principal is the entity that originates the event. This field is mandatory - and can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + and can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). - *Src* The source entity being acted upon. Depending on the `event_type` this field - is optional. Can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + is optional. Can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). - *Target* The target entity being acted upon. Depending on the `event_type` this field - is optional. Can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + is optional. Can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). ### Implementation diff --git a/rfcs/2022-10-12-14742-flexible-metric-tags.md b/rfcs/2022-10-12-14742-flexible-metric-tags.md index ccb0a2340e167..283541491d255 100644 --- a/rfcs/2022-10-12-14742-flexible-metric-tags.md +++ b/rfcs/2022-10-12-14742-flexible-metric-tags.md @@ -127,9 +127,9 @@ supported in the presence of multi-valued tags, which will be selectable with co 1. The individual values are tracked as before. Events are dropped when any one tag's cardinality exceeds the limit, but only the tags that would exceed the limit are dropped. -1. The individual values are tracked as before. Events are dropped when any one tag's cardinality +2. The individual values are tracked as before. Events are dropped when any one tag's cardinality exceeds the limit, and all values of tags that would exceed the limit are dropped. -1. Values of multi-valued tags are combined before tracking. Events are dropped as before and all +3. Values of multi-valued tags are combined before tracking. Events are dropped as before and all values of tags that would exceed the limit are dropped. #### Sinks @@ -215,7 +215,7 @@ The use of an `IndexSet` for the tag value provides us with two useful invariant 1. Only unique values for each tag will be stored, which prevents repeated values from showing up in the output. -1. The values can be retrieved in the order they first appeared, which allows us to trivially +2. The values can be retrieved in the order they first appeared, which allows us to trivially retrieve either the first or last stored value. ## Drawbacks @@ -257,16 +257,16 @@ which will cause problems for users: 1. Unconditionally expose the tags as arrays of values using the existing naming, but still accept assignments using either single values or arrays of values. This will cause breakage to existing scripts that relies on the existing single value tag values. -1. For Lua or VRL scripts, conditionally expose the tags as single values or arrays, as described in +2. For Lua or VRL scripts, conditionally expose the tags as single values or arrays, as described in the proposal, but accept assignments following the native JSON codec scheme. In Lua, this could cause a breaking change where scripts that emit metrics that have the wrong tabs type to be accepted for transmission. In VRL, this would create headaches for type definitions, at best preventing proper validation of programs. -1. Expose the tags as single values using the existing naming, picking some arbitrary value when a +3. Expose the tags as single values using the existing naming, picking some arbitrary value when a tag has multiple values, and set up a secondary tags structure that exposes the arrays. This will lead to all kinds of confusion and conflicts when the same tag is assigned through different variables. -1. Add functions specifically for manipulating tag sets. This continues to make metrics management +4. Add functions specifically for manipulating tag sets. This continues to make metrics management look like a second-class afterthought, and doesn't ease any compatibility problems for existing scripts. @@ -284,8 +284,8 @@ that would support this feature but with different semantics: 1. `Vec` — Retains the ordering of tags as they appear, but allows for duplicate values and cannot support both bare tags and multiple values simultaneously. -1. `Vec>` — Same as above but supports bare tags and mutiple values simultaneously. -1. `BTreeSet>` — Duplicate values are merged but are sorted, likely putting the bare +2. `Vec>` — Same as above but supports bare tags and mutiple values simultaneously. +3. `BTreeSet>` — Duplicate values are merged but are sorted, likely putting the bare tag first for single-value uses. There are also at least two other container types that could possibly support this use case: diff --git a/rfcs/2022-10-31-15056-tooling-revamp.md b/rfcs/2022-10-31-15056-tooling-revamp.md index 70e83db6bb530..d0660f5361222 100644 --- a/rfcs/2022-10-31-15056-tooling-revamp.md +++ b/rfcs/2022-10-31-15056-tooling-revamp.md @@ -41,7 +41,7 @@ This RFC discusses improving Vector's developer tooling in order to ease mainten - Test failures are difficult to debug, esp. since [this change](https://github.com/vectordotdev/vector/pull/13128) - Windows is essentially unsupported since Make takes a great deal of effort to install and most [scripts](https://github.com/vectordotdev/vector/tree/v0.24.2/scripts) require Bash and utilities like `find` -- Adding tests for new integrations to CI ([here](https://github.com/vectordotdev/vector/blob/v0.24.2/Makefile#L333-L341) and [here](https://github.com/vectordotdev/vector/blob/v0.24.2/.github/workflows/integration-test.yml#L58-L91)) is a manual and occasionally forgotten step (see [outstanding questions](#outstanding-questions)) +- Adding tests for new integrations to CI ([Makefile](https://github.com/vectordotdev/vector/blob/v0.24.2/Makefile#L333-L341) and [integration-test workflow](https://github.com/vectordotdev/vector/blob/v0.24.2/.github/workflows/integration-test.yml#L58-L91)) is a manual and occasionally forgotten step (see [outstanding questions](#outstanding-questions)) - Makefiles and scripts can get messy fast and often are hard to scale well ## Proposal diff --git a/rfcs/2023-05-03-data-volume-metrics.md b/rfcs/2023-05-03-data-volume-metrics.md index 368fe874cd420..ba1b571653122 100644 --- a/rfcs/2023-05-03-data-volume-metrics.md +++ b/rfcs/2023-05-03-data-volume-metrics.md @@ -109,7 +109,7 @@ as the O(1) scan. #### `component_received_event_bytes_total` -This metric is emitted by the framework [here][source_sender], so it looks like +This metric is emitted by the [framework's source sender][source_sender], so it looks like the only change needed is to add the service tag. #### `component_sent_event_bytes_total` diff --git a/rfcs/README.md b/rfcs/README.md index 8c0f2a932f5f7..055819c005902 100644 --- a/rfcs/README.md +++ b/rfcs/README.md @@ -78,8 +78,8 @@ more info navigating your solution. 1. Search GitHub for [previous issues](https://github.com/vectordotdev/vector/issues) and [RFCs](https://github.com/vectordotdev/vector/tree/master/rfcs) on this topic. -1. If an RFC issue does not exist, [open one](https://github.com/vectordotdev/vector/issues/new/choose). -1. Use the issue to obtain consensus that an RFC is necessary. +2. If an RFC issue does not exist, [open one](https://github.com/vectordotdev/vector/issues/new/choose). +3. Use the issue to obtain consensus that an RFC is necessary. - The change might be quickly rejected. - The change might be on our long term roadmap and get deferred. - The change might be blocked by other work. @@ -87,22 +87,22 @@ more info navigating your solution. ### Creating an RFC 1. Create a new branch -1. Copy the [`rfcs/_YYYY-MM-DD-issue#-title.md`](rfcs/_YYYY-MM-DD-issue%23-title.md) template with the appropriate +2. Copy the [`rfcs/_YYYY-MM-DD-issue#-title.md`](rfcs/_YYYY-MM-DD-issue%23-title.md) template with the appropriate name. Be sure to use the issue number you created above. (e.g., `rfcs/2020-02-10-445-internal-observability.md`) -1. Fill in your RFC, pay attention the bullets and guidelines. Do not omit any sections. -1. Work with the Vector team to land on a confident solution. Allocate time for code-level spikes if necessary. -1. Submit your RFC as a pull request and tag reviewers for approval. +3. Fill in your RFC, pay attention the bullets and guidelines. Do not omit any sections. +4. Work with the Vector team to land on a confident solution. Allocate time for code-level spikes if necessary. +5. Submit your RFC as a pull request and tag reviewers for approval. ### Getting an RFC accepted 1. Schedule a "last call" meeting for your RFC. This should be 1 week after opening your pull request. The purpose is to efficiently obtain consensus. -1. At least 3 Vector team members must approve your RFC in the form of pull request approvals. -1. Once approved, self-merge your RFC, or ask a Vector team member to do it for you. +2. At least 3 Vector team members must approve your RFC in the form of pull request approvals. +3. Once approved, self-merge your RFC, or ask a Vector team member to do it for you. ### Implementing an RFC 1. Create issues from the "Plan Of Attack" section. Place them in an epic if necessary. -1. Coordinate with leadership to schedule your work. +2. Coordinate with leadership to schedule your work. ## FAQ diff --git a/vdev/src/commands/check/markdown.rs b/vdev/src/commands/check/markdown.rs index 002ed62908444..3ecf0c91f3201 100644 --- a/vdev/src/commands/check/markdown.rs +++ b/vdev/src/commands/check/markdown.rs @@ -14,16 +14,10 @@ impl Cli { return Ok(()); } - let args: Vec<&str> = vec![ - "--config", - "scripts/.markdownlintrc", - // We should fix these as well. Previously these files were not linted. - "--ignore", - ".github", - ] - .into_iter() - .chain(files.iter().map(String::as_str)) - .collect(); + let args: Vec<&str> = vec!["--config", ".markdownlint.jsonc"] + .into_iter() + .chain(files.iter().map(String::as_str)) + .collect(); app::exec("markdownlint", &args, true) } diff --git a/website/README.md b/website/README.md index 214ed0d4007e2..47475945dbf9d 100644 --- a/website/README.md +++ b/website/README.md @@ -139,8 +139,8 @@ When you make changes to the Markdown sources, Sass/CSS, or JavaScript, the site ### Add a new version of Vector 1. Add the new version to the `versions` list in [`cue/reference/versions.cue`](./cue/reference/versions.cue). Make sure to preserve reverse ordering. -1. Generate a new CUE file for the release by running `make release-prepare` in the root directory of the Vector repo. This generates a CUE file at `cue/releases/{VERSION}.cue`. -1. Add a new Markdown file to [`content/en/releases`](./content/en/releases), where the filename is `{version}.md` (e.g. `0.12.0.md`) and the file has metadata that looks like this: +2. Generate a new CUE file for the release by running `make release-prepare` in the root directory of the Vector repo. This generates a CUE file at `cue/releases/{VERSION}.cue`. +3. Add a new Markdown file to [`content/en/releases`](./content/en/releases), where the filename is `{version}.md` (e.g. `0.12.0.md`) and the file has metadata that looks like this: ```markdown --- diff --git a/website/content/en/blog/adaptive-request-concurrency.md b/website/content/en/blog/adaptive-request-concurrency.md index 9c9aff9597454..b2a701433d638 100644 --- a/website/content/en/blog/adaptive-request-concurrency.md +++ b/website/content/en/blog/adaptive-request-concurrency.md @@ -25,7 +25,7 @@ TRACE tower_limit::rate::service: rate limit exceeded, disabling service Users typically have two questions about this: 1. What does it mean? -1. How can I fix it? +2. How can I fix it? The answer to the first question is simple: Vector has _internally_ rate-limited processing to respect user-configured limits—[`request.rate_limit_duration_secs`][rate_limit_duration_secs] and [`request.rate_limit_num`][rate_limit_num]—for that particular [sink][sinks]. In other words, Vector has intentionally reduced performance to stay within static limits. @@ -66,12 +66,12 @@ We feel strongly that Vector's **Adaptive Request Concurrency** (ARC) feature pr Here's how that plays out in some example scenarios: -Change | | Response -:------|:-:|:-------- -**You deploy more Vector instances** | ➔ |Vector automatically redistributes HTTP throughput across both current and new instances -**You scale up your Elasticsearch cluster** | ➔ | Vector automatically increases concurrency to take full advantage of the new capacity -**You scale your Elasticsearch cluster back down** | ➔ | Vector lowers concurrency to avoid any risk of destabilizing the cluster (while still taking full of advantage of the now-decreased bandwidth) -**Your Elasticsearch cluster experiences a temporary outage** | ➔ |Vector lowers concurrency dramatically and provides backpressure by [buffering][buffer] events +| Change | | Response | +| :------- | :-: | :-------- | +| **You deploy more Vector instances** | ➔ | Vector automatically redistributes HTTP throughput across both current and new instances | +| **You scale up your Elasticsearch cluster** | ➔ | Vector automatically increases concurrency to take full advantage of the new capacity | +| **You scale your Elasticsearch cluster back down** | ➔ | Vector lowers concurrency to avoid any risk of destabilizing the cluster (while still taking full of advantage of the now-decreased bandwidth) | +| **Your Elasticsearch cluster experiences a temporary outage** | ➔ | Vector lowers concurrency dramatically and provides backpressure by [buffering][buffer] events | With ARC, these scenarios require no human intervention. Vector quietly hums along making these decisions for you with a speed and granularity that rate limits simply cannot provide. diff --git a/website/content/en/blog/highlights-february-2025.md b/website/content/en/blog/highlights-february-2025.md index 1adf7bce8787e..eabb4a8adda38 100644 --- a/website/content/en/blog/highlights-february-2025.md +++ b/website/content/en/blog/highlights-february-2025.md @@ -44,5 +44,5 @@ We supported generating a Vector config schema for a while. This guide demonstra popular IDEs in order to help with writing configs. We would love to this to VRL IDE support in the future. In the meantime, we would like to highlight -this [tree-sitter](https://github.com/tree-sitter/tree-sitter) plugin that was developed by https://github.com/belltoy. You can read more -[here](https://github.com/vectordotdev/vrl/issues/964). +this [tree-sitter](https://github.com/tree-sitter/tree-sitter) plugin that was developed by https://github.com/belltoy. You can read more about it in the +[VRL tree-sitter issue](https://github.com/vectordotdev/vrl/issues/964). diff --git a/website/content/en/blog/how-we-test-vector.md b/website/content/en/blog/how-we-test-vector.md index 2d1313ec0a66e..cd029c3a5116a 100644 --- a/website/content/en/blog/how-we-test-vector.md +++ b/website/content/en/blog/how-we-test-vector.md @@ -21,9 +21,9 @@ task for software like Vector: 1. It's relatively new and not as "battle tested" as more widely deployed software. -1. The vast majority of its functionality lives at the edges, interfacing with +2. The vast majority of its functionality lives at the edges, interfacing with various external systems. -1. Instead of a monolithic application designed for a single task, it is +3. Instead of a monolithic application designed for a single task, it is a collection of components that can be assembled into a near-infinite number of configurations. diff --git a/website/content/en/blog/tracking-allocations.md b/website/content/en/blog/tracking-allocations.md index 009ff33622c11..7fd2815af22b9 100644 --- a/website/content/en/blog/tracking-allocations.md +++ b/website/content/en/blog/tracking-allocations.md @@ -44,4 +44,4 @@ In our development and testing of this feature, we've observed ~20% reduction in We currently do not provide support for tracking memory ownership between components. For example, when a sink is batching events, and is configured to batch many events, you may observe high Vector memory usage. If you looked at the memory usage metrics, you would see most of it being attributed to components that either created the events (such as a source) or processed the events (such as any transforms that modified the event) rather than the sink which is batching the events. Adding support for shared ownership tracking provides further insights into the lifetimes of components, further easing the debugging process. -Please Let us know your feedback and suggestions [here](https://github.com/vectordotdev/vector/issues/15474)! +Please let us know your feedback and suggestions in the [allocation tracking feedback issue](https://github.com/vectordotdev/vector/issues/15474)! diff --git a/website/content/en/blog/vector-remap-language.md b/website/content/en/blog/vector-remap-language.md index d580e612ac4bb..368b113afbb3f 100644 --- a/website/content/en/blog/vector-remap-language.md +++ b/website/content/en/blog/vector-remap-language.md @@ -330,14 +330,14 @@ maintaining flexibility. This makes VRL ideal for always-up, performance-sensitive infrastructure like observability pipelines. To illustrate how we achieve this, below is a VRL feature matrix across these two principles: -| Feature | Safety | Performance | -|:--------------------------|:------:|:-----------:| -| [Progressive type safety] | ✅ | | -| [Fail safety] | ✅ | | -| [Memory safety] | ✅ | | -| [Ergonomic safety] | ✅ | ✅ | -| [Vector/Rust native] | ✅ | ✅ | -| [Stateless] | ✅ | ✅ | +| Feature | Safety | Performance | +| :-- | :----: | :---------: | +| [Progressive type safety] | ✅ | | +| [Fail safety] | ✅ | | +| [Memory safety] | ✅ | | +| [Ergonomic safety] | ✅ | ✅ | +| [Vector/Rust native] | ✅ | ✅ | +| [Stateless] | ✅ | ✅ | For more info on each click on the feature, but for the purposes of demonstrating how VRL is unique, let's touch on the first two: *progressive type @@ -411,7 +411,7 @@ either specify the type of the `.log` field or handle the error in the event that `.log` is not a string. To resolve this error, the user must do one of three things: -1. **Handle the error** +1. **Handle the error** ```coffee ., err = parse_common_log(.log) @@ -430,7 +430,7 @@ three things: handling this error is a *very* common mistake that would otherwise result in data loss and downtime. -2. **Raise the error and abort** +2. **Raise the error and abort** ```coffee . = parse_common_log!(.log) @@ -445,7 +445,7 @@ three things: operators. Again, this forces the users to decide how to handle errors instead of being surprised by them. -3. **Specify types** +3. **Specify types** ```coffee .log = to_string!(.log) diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index c1eb175139124..dff4be7153bbf 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -179,11 +179,11 @@ docker restart -f $(docker ps -aqf "name=vector") The commands above involve configuring Vector using TOML, but you can also use JSON or YAML. You can also use one of three image variants (the commands assume `alpine`): -Variant | Image basis -:-------|:----------- -`alpine` | [Alpine](https://hub.docker.com/_/alpine), a Linux distro built around [musl libc](https://www.musl-libc.org) and [BusyBox](https://busybox.net) -`debian` | The [`debian-slim`](https://hub.docker.com/_/debian) image, which is a smaller and more compact version of the standard `debian` image -`distroless` | The [Distroless](https://github.com/GoogleContainerTools/distroless) project, which provides extremely lean images with no package managers, shells, or other inessential utilities +| Variant | Image basis | +| :-------- | :----------- | +| `alpine` | [Alpine](https://hub.docker.com/_/alpine), a Linux distro built around [musl libc](https://www.musl-libc.org) and [BusyBox](https://busybox.net) | +| `debian` | The [`debian-slim`](https://hub.docker.com/_/debian) image, which is a smaller and more compact version of the standard `debian` image | +| `distroless` | The [Distroless](https://github.com/GoogleContainerTools/distroless) project, which provides extremely lean images with no package managers, shells, or other inessential utilities | ### Helm diff --git a/website/content/en/docs/administration/monitoring.md b/website/content/en/docs/administration/monitoring.md index d9947a1b55e67..76819ee25ab18 100644 --- a/website/content/en/docs/administration/monitoring.md +++ b/website/content/en/docs/administration/monitoring.md @@ -77,14 +77,14 @@ sinks: Vector logs at the `info` level by default. You can set a different level when [starting] up your instance using either command-line flags or the `VECTOR_LOG` environment variable. The table below details these options: -Method | Description -:------|:----------- -`-v` flag | Drops the log level to `debug` -`-vv` flag | Drops the log level to `trace` -`-q` flag | Raises the log level to `warn` -`-qq` flag | Raises the log level to `error` -`-qqq` flag | Disables logging -`VECTOR_LOG=` environment variable | Set the log level. Must be one of `trace`, `debug`, `info`, `warn`, `error`, `off`. +| Method | Description | +| :------- | :----------- | +| `-v` flag | Drops the log level to `debug` | +| `-vv` flag | Drops the log level to `trace` | +| `-q` flag | Raises the log level to `warn` | +| `-qq` flag | Raises the log level to `error` | +| `-qqq` flag | Disables logging | +| `VECTOR_LOG=` environment variable | Set the log level. Must be one of `trace`, `debug`, `info`, `warn`, `error`, `off`. | #### Stack traces diff --git a/website/content/en/docs/administration/validating.md b/website/content/en/docs/administration/validating.md index bfdaec1f98189..0519d971fced4 100644 --- a/website/content/en/docs/administration/validating.md +++ b/website/content/en/docs/administration/validating.md @@ -36,10 +36,10 @@ files, including: These checks verify that the configuration file contains a valid topology: 1. At least one [source][sources] is defined -1. At least one [sink][sinks] is defined -1. All inputs for each topology component (specified using the `inputs` parameter) contain at least +2. At least one [sink][sinks] is defined +3. All inputs for each topology component (specified using the `inputs` parameter) contain at least one value. -1. All inputs refer to valid and upstream [sources] or [transforms]. +4. All inputs refer to valid and upstream [sources] or [transforms]. ### Environment checks @@ -47,7 +47,7 @@ Finally, these checks ensure that Vector is running in an environment that can s configured topology: 1. All components have the pre-requisites to run, e.g. data directories exist and are writable. -1. All sinks can connect to their specified targets. +2. All sinks can connect to their specified targets. These environment checks can be disabled using the [`--no-environment`][no_environment] flag: diff --git a/website/content/en/docs/introduction/concepts.md b/website/content/en/docs/introduction/concepts.md index ad2dadddc6135..237b4fa903d50 100644 --- a/website/content/en/docs/introduction/concepts.md +++ b/website/content/en/docs/introduction/concepts.md @@ -40,7 +40,7 @@ A **trace** event can be thought of as a special kind of log event. The componen **Note**: Support for traces is limited and is in alpha. If you’re interested in using traces with a Vector component that doesn’t yet support them, -please check the list of open issues [here](https://github.com/vectordotdev/vector/issues?q=is%3Aissue+state%3Aopen+label%3A%22domain%3A%20traces%22). +please check the [list of open traces issues](https://github.com/vectordotdev/vector/issues?q=is%3Aissue+state%3Aopen+label%3A%22domain%3A%20traces%22). If you don’t see your use case covered, feel free to [open a new issue](https://github.com/vectordotdev/vector/issues/new?template=feature.yml). @@ -88,7 +88,7 @@ This is the default behavior. When a buffer fills up, backpressure will be appli `buffer.when_full = drop_newest` When a buffer fills up, new events will be dropped. This does _not_ provide backpressure. -View the full configuration options for buffers [here](/docs/reference/configuration/sinks/vector/#buffer). +View the [full buffer configuration options](/docs/reference/configuration/sinks/vector/#buffer). ## Backpressure diff --git a/website/content/en/docs/reference/configuration/unit-tests.md b/website/content/en/docs/reference/configuration/unit-tests.md index c838086f50286..5e11240c9136f 100644 --- a/website/content/en/docs/reference/configuration/unit-tests.md +++ b/website/content/en/docs/reference/configuration/unit-tests.md @@ -14,9 +14,9 @@ work just like unit tests in most programming languages: 1. Provide a set of [inputs](#inputs) to a transform (or to [multiple transforms](#multiple) chained together). -1. Specify the expected [outputs](#outputs) from the changes made by the transform (or multiple +2. Specify the expected [outputs](#outputs) from the changes made by the transform (or multiple transforms). -1. Receive directly actionable feedback from any test failures. +3. Receive directly actionable feedback from any test failures. Unit tests can serve as a useful guardrail when running in Vector in production settings where you need to ensure that your topology doesn't exhibit unexpected behavior and generally improve the @@ -219,14 +219,14 @@ Inside each test definition, you need to specify two things: In the `inputs` array for the test, you have these options: -Parameter | Type | Description -:---------|:-----|:----------- -`type` | string | The type of input you're providing. [`vrl`](#logs), [`log`](#logs), [`raw`](#logs), or [`metric`](#metrics) are currently the only valid values. -`insert_at` | string (name of transform) | The name of the transform into which the test input is inserted. This is particularly useful when you want to test only a subset of a transform pipeline. -`value` | string (raw event value) | A raw string value to act as an input event. Use only in cases where events are raw strings and not structured objects with event fields. -`log_fields` | object | If the transform handles [log events](#logs), these are the key/value pairs that comprise the input event. -`metric` | object | If the transform handles [metric events](#metrics), these are the fields that comprise that metric. Subfields include `name`, `tags`, `kind`, and others. -`source` | string (vrl program) | If the transform handles [log events](#logs), the result of the vrl program will be the input event. +| Parameter | Type | Description | +| :--------- | :---- | :----------- | +| `type` | string | The type of input you're providing. [`vrl`](#logs), [`log`](#logs), [`raw`](#logs), or [`metric`](#metrics) are currently the only valid values. | +| `insert_at` | string (name of transform) | The name of the transform into which the test input is inserted. This is particularly useful when you want to test only a subset of a transform pipeline. | +| `value` | string (raw event value) | A raw string value to act as an input event. Use only in cases where events are raw strings and not structured objects with event fields. | +| `log_fields` | object | If the transform handles [log events](#logs), these are the key/value pairs that comprise the input event. | +| `metric` | object | If the transform handles [metric events](#metrics), these are the fields that comprise that metric. Subfields include `name`, `tags`, `kind`, and others. | +| `source` | string (vrl program) | If the transform handles [log events](#logs), the result of the vrl program will be the input event. | Here's an example `inputs` declaration: @@ -248,17 +248,17 @@ message = "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Somet In the `outputs` array of your unit testing configuration, you specify two things: -Parameter | Type | Description -:---------|:-----|:----------- -`extract_from` | string (name of transform) | The transform whose output you want to test. -`conditions` | array of objects | The [VRL conditions](#verifying) to run against the output. +| Parameter | Type | Description | +| :--------- | :---- | :----------- | +| `extract_from` | string (name of transform) | The transform whose output you want to test. | +| `conditions` | array of objects | The [VRL conditions](#verifying) to run against the output. | Each condition in the `conditions` array has two fields: -Parameter | Type | Description -:---------|:-----|:----------- -`type` | string | The type of condition you're providing. [`vrl`][vrl] is currently the only valid value. -`source` | string (VRL Boolean expression) | Explained in detail [above](#verifying). +| Parameter | Type | Description | +| :--------- | :---- | :----------- | +| `type` | string | The type of condition you're providing. [`vrl`][vrl] is currently the only valid value. | +| `source` | string (VRL Boolean expression) | Explained in detail [above](#verifying). | Here's an example `outputs` declaration: diff --git a/website/content/en/docs/reference/vrl/_index.md b/website/content/en/docs/reference/vrl/_index.md index a0b2cfb426efe..9685d6eddfdbf 100644 --- a/website/content/en/docs/reference/vrl/_index.md +++ b/website/content/en/docs/reference/vrl/_index.md @@ -166,14 +166,14 @@ flexibility. This makes VRL ideal for critical, performance-sensitive infrastructure, like observability pipelines. To illustrate how we achieve these, below is a VRL feature matrix across these principles: -| Feature | Safety | Performance | -| :-------------------------------------------- | :----- | :---------- | -| [Compilation](#compilation) | ✅ | ✅ | -| [Ergonomic safety](#ergonomic-safety) | ✅ | ✅ | -| [Fail safety](#fail-safety) | ✅ | | -| [Memory safety](#memory-safety) | ✅ | | -| [Vector and Rust native](#vector-rust-native) | ✅ | ✅ | -| [Statelessness](#stateless) | ✅ | ✅ | +| Feature | Safety | Performance | +| :-- | :----- | :---------- | +| [Compilation](#compilation) | ✅ | ✅ | +| [Ergonomic safety](#ergonomic-safety) | ✅ | ✅ | +| [Fail safety](#fail-safety) | ✅ | | +| [Memory safety](#memory-safety) | ✅ | | +| [Vector and Rust native](#vector-rust-native) | ✅ | ✅ | +| [Statelessness](#stateless) | ✅ | ✅ | ## Concepts diff --git a/website/content/en/docs/reference/vrl/errors.md b/website/content/en/docs/reference/vrl/errors.md index e6da9cbec8245..ba6fe86ae6d88 100644 --- a/website/content/en/docs/reference/vrl/errors.md +++ b/website/content/en/docs/reference/vrl/errors.md @@ -64,17 +64,17 @@ of an infallible assignment. ##### Empty values -Type | Empty value -:----|:----------- -String | `""` -Integer | `0` -Float | `0.0` -Boolean | `false` -Object | `{}` -Array | `[]` -Timestamp | `t'1970-01-01T00:00:00Z'` (Unix epoch) -Regular expression | `r''` -Null | `null` +| Type | Empty value | +| :---- | :----------- | +| String | `""` | +| Integer | `0` | +| Float | `0.0` | +| Boolean | `false` | +| Object | `{}` | +| Array | `[]` | +| Timestamp | `t'1970-01-01T00:00:00Z'` (Unix epoch) | +| Regular expression | `r''` | +| Null | `null` | #### Coalescing diff --git a/website/content/en/docs/setup/going-to-prod/arch/aggregator.md b/website/content/en/docs/setup/going-to-prod/arch/aggregator.md index 3bf337c7f4ac6..6211e8b5f14c9 100644 --- a/website/content/en/docs/setup/going-to-prod/arch/aggregator.md +++ b/website/content/en/docs/setup/going-to-prod/arch/aggregator.md @@ -111,4 +111,4 @@ This eliminates the need to deploy a single monolith aggregator, creating an unn ## Support -For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about that product [here](https://www.datadoghq.com/product/observability-pipelines/). +For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about [Datadog Observability Pipelines](https://www.datadoghq.com/product/observability-pipelines/). diff --git a/website/content/en/docs/setup/going-to-prod/arch/unified.md b/website/content/en/docs/setup/going-to-prod/arch/unified.md index 59be6fa2a779a..710d1d4bdc6a5 100644 --- a/website/content/en/docs/setup/going-to-prod/arch/unified.md +++ b/website/content/en/docs/setup/going-to-prod/arch/unified.md @@ -23,4 +23,4 @@ We recommend this architecture for Vector users that have already deployed the a ## Support -For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about that [here](https://www.datadoghq.com/product/observability-pipelines/). +For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about [Datadog Observability Pipelines](https://www.datadoghq.com/product/observability-pipelines/). diff --git a/website/content/en/docs/setup/installation/manual/from-source.md b/website/content/en/docs/setup/installation/manual/from-source.md index 5ad19d46bce5c..0af130387f999 100644 --- a/website/content/en/docs/setup/installation/manual/from-source.md +++ b/website/content/en/docs/setup/installation/manual/from-source.md @@ -235,7 +235,7 @@ To update Vector, follow the same [installation](#installation) instructions abo Vector supports many feature flags to customize which features are included in a build. By default, all sources, transforms, and sinks are enabled. To view a complete list of features, they are listed -under "[features]" [here](https://github.com/vectordotdev/vector/blob/master/Cargo.toml). +under "[features]" in [Cargo.toml](https://github.com/vectordotdev/vector/blob/master/Cargo.toml). Example of building with only specific components: diff --git a/website/content/en/docs/setup/installation/package-managers/helm.md b/website/content/en/docs/setup/installation/package-managers/helm.md index 87c49e56140cb..0c38f086781fd 100644 --- a/website/content/en/docs/setup/installation/package-managers/helm.md +++ b/website/content/en/docs/setup/installation/package-managers/helm.md @@ -27,7 +27,7 @@ To check available Helm chart configuration options: helm show values vector/vector ``` -This example configuration file deploys Vector as an Agent, the full default configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [configuration] docs page. +This example configuration file deploys Vector as an Agent, the [full default configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [configuration] docs page. ```yaml cat <<-'VALUES' > values.yaml @@ -69,7 +69,7 @@ To check available Helm chart configuration options: helm show values vector/vector ``` -The chart deploys an Aggregator by default, the full configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [Configuration] docs page. +The chart deploys an Aggregator by default, the [full configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [Configuration] docs page. ### Installing diff --git a/website/content/en/docs/setup/installation/platforms/kubernetes.md b/website/content/en/docs/setup/installation/platforms/kubernetes.md index b86e3ad251e48..8629db632ca19 100644 --- a/website/content/en/docs/setup/installation/platforms/kubernetes.md +++ b/website/content/en/docs/setup/installation/platforms/kubernetes.md @@ -40,7 +40,7 @@ kubectl create namespace --dry-run=client -o yaml vector > namespace.yaml ##### Prepare your kustomization file -This example configuration file deploys Vector as an Agent, the full default configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [configuration] docs page. +This example configuration file deploys Vector as an Agent, the [full default configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [configuration] docs page. ```shell cat <<-'KUSTOMIZATION' > kustomization.yaml @@ -99,7 +99,7 @@ kubectl create namespace --dry-run=client -o yaml vector > namespace.yaml ##### Prepare your kustomization file -This example configuration deploys Vector as an Aggregator, the full configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [Configuration] docs page. +This example configuration deploys Vector as an Aggregator, the [full configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [Configuration] docs page. ```shell cat <<-'KUSTOMIZATION' > kustomization.yaml diff --git a/website/content/en/guides/developer/config-autocompletion.md b/website/content/en/guides/developer/config-autocompletion.md index e092a023f20f5..d98486e1c4ae2 100644 --- a/website/content/en/guides/developer/config-autocompletion.md +++ b/website/content/en/guides/developer/config-autocompletion.md @@ -37,11 +37,11 @@ vector generate-schema -o vector-v0.45.0-schema.json 1. `Settings | Languages & Frameworks | Schemas and DTDs | JSON Schema Mappings` 2. Import `vector-v0.45.0-schema.json` -You can find more details [here][jetbrains]. +You can find more details in the [JetBrains JSON schema documentation][jetbrains]. ### Visual Studio Code -Follow the guide [here][vscode]. +Follow the [VS Code YAML schema validation guide][vscode]. ## Example diff --git a/website/content/en/guides/developer/debugging.md b/website/content/en/guides/developer/debugging.md index 4e84514264e02..af1081377f2d5 100644 --- a/website/content/en/guides/developer/debugging.md +++ b/website/content/en/guides/developer/debugging.md @@ -35,7 +35,7 @@ You can set different verbosity levels for specific components: VECTOR_LOG=info,vector::sources::aws_s3=warn vector --config path/to/config.yaml ``` -You can find more information on the syntax [here](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#usage-notes). +You can find more information on the [EnvFilter syntax](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#usage-notes). ### Vector Tools diff --git a/website/content/en/highlights/2020-10-27-kubernetes-integration.md b/website/content/en/highlights/2020-10-27-kubernetes-integration.md index b94d1eb296e19..505f86f217c55 100644 --- a/website/content/en/highlights/2020-10-27-kubernetes-integration.md +++ b/website/content/en/highlights/2020-10-27-kubernetes-integration.md @@ -23,14 +23,14 @@ Kubernetes observability data. ## Feature highlights -1. [**A new `kubernetes_logs` source**][kubernetes_logs_source] - A new source +1. [**A new `kubernetes_logs` source**][kubernetes_logs_source] - A new source designed to handle the intricacies of Kubernetes log collection. It'll collect all Pod logs, merge split logs together, and enrich them with k8s metadata. -2. [**YAML config support**][config_formats_highlight] - +2. [**YAML config support**][config_formats_highlight] - To ensure Vector fits cleanly into your existing K8s workflows, Vector now accepts YAML and JSON config formats. -3. [**Adaptive Request Currency (ARC)**][adaptive_concurrency_post] - +3. [**Adaptive Request Currency (ARC)**][adaptive_concurrency_post] - A new Vector feature designed to automatically optimize HTTP communication in the face of ever changing environments like Kubernetes. It does away with static rate limits and raises the performance and reliability of your entire diff --git a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md index 760d069821723..604b85bfc4bf7 100644 --- a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md +++ b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md @@ -66,7 +66,7 @@ Kubernetes where Prometheus is tightly integrated. To get started, setup the new [`prometheus_remote_write` source][prometheus_remote_write_source] and send your metrics to [Datadog][datadog], [New Relic][new_relic], [Influx][influx], -[Elasticsearch][elastic], and [more][sinks]: +[Elasticsearch][elastic], and [many other sinks][sinks]: ```toml title="vector.toml" [sources.prometheus] diff --git a/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md b/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md index dc37a0f044e8d..4c51c5dba4eb8 100644 --- a/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md +++ b/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md @@ -13,12 +13,12 @@ badges: 0.11 includes some minor breaking changes: 1. [The metrics emitted by the `internal_metrics` source have changed names.](#second) -1. [The `statsd` sink now supports all socket types.](#third) -1. [The `source_type` field is now explicit in the `splunk_hec` sink.](#fifth) -1. [Remove forwarding to syslog from distributed systemd unit.](#sixth) -1. [The `http` source no longer dedots JSON fields.](#seventh) -1. [The `prometheus` sink has been renamed to `prometheus_exporter`](#first) -1. [The `reduce` transform `identifier_fields` was renamed to `group_by`.](#fourth) +2. [The `statsd` sink now supports all socket types.](#third) +3. [The `source_type` field is now explicit in the `splunk_hec` sink.](#fifth) +4. [Remove forwarding to syslog from distributed systemd unit.](#sixth) +5. [The `http` source no longer dedots JSON fields.](#seventh) +6. [The `prometheus` sink has been renamed to `prometheus_exporter`](#first) +7. [The `reduce` transform `identifier_fields` was renamed to `group_by`.](#fourth) We cover each below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md index e0e21c3bba265..9f05bda354d2f 100644 --- a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md +++ b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md @@ -14,10 +14,10 @@ badges: painlessly. If you have questions, [hop in our chat][chat] and we'll help you upgrade. 1. [Breaking: The `encoding.codec` option is now required for all relevant sinks](#first) -1. [Breaking: Vector `check_fields` conditions now require the `type` option](#second) -1. [Breaking: The `generator` source requires a `format` option](#third) -1. [Deprecation: Many transforms have been deprecated in favor of the new `remap` transform](#fourth) -1. [Deprecation: The `file` source `start_at_beginning` has been deprecated](#fifth) +2. [Breaking: Vector `check_fields` conditions now require the `type` option](#second) +3. [Breaking: The `generator` source requires a `format` option](#third) +4. [Deprecation: Many transforms have been deprecated in favor of the new `remap` transform](#fourth) +5. [Deprecation: The `file` source `start_at_beginning` has been deprecated](#fifth) ## Upgrade Guide diff --git a/website/content/en/highlights/2021-08-24-vector-aggregator.md b/website/content/en/highlights/2021-08-24-vector-aggregator.md index c55f7aecbd19e..95a2c790e3b69 100644 --- a/website/content/en/highlights/2021-08-24-vector-aggregator.md +++ b/website/content/en/highlights/2021-08-24-vector-aggregator.md @@ -37,7 +37,7 @@ installing a HAProxy Deployment for load balancing across multiple aggregators. ## Setup and installation -Instructions on how setup and install the `vector-aggregator` chart can be found [here][setup]. +Instructions on how setup and install the `vector-aggregator` chart can be found in the [Helm installation guide][setup]. ## Looking forward diff --git a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md index 9061612d5d8b2..96c30410f344c 100644 --- a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md +++ b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md @@ -13,10 +13,10 @@ badges: Vector's 0.16.0 release includes three **breaking changes**: 1. [Component name field renamed to ID](#name-to-id) -1. [Datadog Log sink encoding option removed](#encoding) -1. [Renaming of `memory_use_bytes` internal metric](#memory_use_bytes) -1. [`datadog_logs` source renamed to `datadog_agent`](#datadog_logs_rename) -1. [`kubernetes_logs` source's new RBAC](#kubernetes_logs_rbac) +2. [Datadog Log sink encoding option removed](#encoding) +3. [Renaming of `memory_use_bytes` internal metric](#memory_use_bytes) +4. [`datadog_logs` source renamed to `datadog_agent`](#datadog_logs_rename) +5. [`kubernetes_logs` source's new RBAC](#kubernetes_logs_rbac) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md b/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md index 2af26a0c1b6d5..a6cf5ba705354 100644 --- a/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md +++ b/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md @@ -13,12 +13,12 @@ badges: Vector's 0.17.0 release includes several **breaking changes**: 1. [Blackhole sink configuration changes](#blackhole) -1. [Datadog Logs sink loses `batch.max_bytes` setting](#datadog_logs_max_bytes) -1. [Vector now logs to stderr](#logging) -1. [The `generator` source now has a default `interval` setting](#interval) -1. [The deprecated `wasm` transform was removed](#wasm) -1. [The `exec` source now has a `decoding` setting](#exec_source) -1. [The algorithm underlying ARC has been optimized](#arc) +2. [Datadog Logs sink loses `batch.max_bytes` setting](#datadog_logs_max_bytes) +3. [Vector now logs to stderr](#logging) +4. [The `generator` source now has a default `interval` setting](#interval) +5. [The deprecated `wasm` transform was removed](#wasm) +6. [The `exec` source now has a `decoding` setting](#exec_source) +7. [The algorithm underlying ARC has been optimized](#arc) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md b/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md index 66c49480594a4..2270990762970 100644 --- a/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md +++ b/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md @@ -13,10 +13,10 @@ badges: Vector's 0.18.0 release includes **breaking changes**: 1. [`batch.max_size` no longer valid for sinks](#batch-max-size) -1. [`request.in_flight_limit` no longer valid for sources and sinks](#request-in-flight-limit) -1. [`http_client_responses_total` now labels status with only numeric code](#http_client_responses_total) -1. [Field name change for aggregated summaries in `metric_to_log` transform](#agg_summary_metric_to_log) -1. [Removal of deprecated configuration fields for the Datadog metrics sink: `host` and `namespace`](#dd_metrics_sink_deprecated_fields) +2. [`request.in_flight_limit` no longer valid for sources and sinks](#request-in-flight-limit) +3. [`http_client_responses_total` now labels status with only numeric code](#http_client_responses_total) +4. [Field name change for aggregated summaries in `metric_to_log` transform](#agg_summary_metric_to_log) +5. [Removal of deprecated configuration fields for the Datadog metrics sink: `host` and `namespace`](#dd_metrics_sink_deprecated_fields) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md index fcfa8c9c8aff7..e23e7b54c9f5e 100644 --- a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md +++ b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md @@ -13,9 +13,9 @@ badges: Vector's 0.19.0 release includes **breaking changes**: 1. [Removal of deprecated configuration fields for the Splunk HEC Logs sink: `host`](#splunk-hec-logs-sink-deprecated-fields) -1. [Updated internal metrics for the Splunk HEC sinks](#splunk-hec-sinks-metrics-update) -1. [Removal of deprecated configuration fields for the Elasticsearch sink](#elasticsearch-sink-deprecated-fields) -1. [Removal of default for `version` field for Vector source and sink](#vector-version) +2. [Updated internal metrics for the Splunk HEC sinks](#splunk-hec-sinks-metrics-update) +3. [Removal of deprecated configuration fields for the Elasticsearch sink](#elasticsearch-sink-deprecated-fields) +4. [Removal of default for `version` field for Vector source and sink](#vector-version) And **deprecations**: diff --git a/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md b/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md index 9bb01a8f6026a..997aefb340c72 100644 --- a/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md @@ -13,12 +13,12 @@ badges: Vector's 0.20.0 release includes **breaking changes**: 1. [Change to set expiration behavior in `prometheus_exporter` sink](#prom-exporter-set-expiration) -1. [New metrics behavior for `route` transform](#metrics-route-transform) -1. [New internal events for `loki` sink](#events-loki-sink) -1. [`if` statement predicates in `VRL` need to handle errors](#vrl-fallible-predicates) -1. [Division by a literal nonzero number in `VRL` is no longer fallible](#vrl-infallible-division) -1. ['syslog' decoder will raise an error if the incoming message cannot be decoded successfully](#syslog-decoder) -1. [Unit tests no longer allow nonexistent targets in `extract_from`/`no_outputs_from` configurations](#unit-test-nonexistent-build-error) +2. [New metrics behavior for `route` transform](#metrics-route-transform) +3. [New internal events for `loki` sink](#events-loki-sink) +4. [`if` statement predicates in `VRL` need to handle errors](#vrl-fallible-predicates) +5. [Division by a literal nonzero number in `VRL` is no longer fallible](#vrl-infallible-division) +6. ['syslog' decoder will raise an error if the incoming message cannot be decoded successfully](#syslog-decoder) +7. [Unit tests no longer allow nonexistent targets in `extract_from`/`no_outputs_from` configurations](#unit-test-nonexistent-build-error) And **deprecations**: diff --git a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md index 855cfa2ddd81a..761e6d861455b 100644 --- a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md @@ -58,10 +58,10 @@ Here are some examples that require migrating | old syntax | new syntax | comment | | ------------- | --------------- | -------------- | -| foo\\.bar.baz | "foo.bar".baz | The `.` field separator needs to be escaped if used as part of a field name. The old syntax allowed individual character escaping. The new syntax requires quotes around the field name. -| headers.User-Agent | headers."User-Agent" | `-` requires quotes with the new syntax -| foo with spaces | "foo with spaces" | Spaces also need to be quoted -| foo\\"bar | "foo\\"bar" | Double quotes and backlashes must be escaped _inside_ quotes +| foo\\.bar.baz | "foo.bar".baz | The `.` field separator needs to be escaped if used as part of a field name. The old syntax allowed individual character escaping. The new syntax requires quotes around the field name. | +| headers.User-Agent | headers."User-Agent" | `-` requires quotes with the new syntax | +| foo with spaces | "foo with spaces" | Spaces also need to be quoted | +| foo\\"bar | "foo\\"bar" | Double quotes and backlashes must be escaped _inside_ quotes | ### TOML transform example @@ -84,7 +84,7 @@ inputs = ["input"] fields.match = ["message.\"user-identifier\""] ``` -For more information on the new syntax, you can review the documentation [here](https://vector.dev/docs/reference/vrl/expressions/#path) +For more information on the new syntax, you can review the [VRL path expressions documentation](https://vector.dev/docs/reference/vrl/expressions/#path) diff --git a/website/content/en/highlights/2022-03-31-native-event-codecs.md b/website/content/en/highlights/2022-03-31-native-event-codecs.md index b31b000e6bdfa..0a10938afffea 100644 --- a/website/content/en/highlights/2022-03-31-native-event-codecs.md +++ b/website/content/en/highlights/2022-03-31-native-event-codecs.md @@ -63,9 +63,9 @@ a `timestamp` key like: ``` The specific JSON schema here is subject to change, but you can find an initial -schema [here][cue schema]. The protobuf schema for the `native` codec is the -same used in the `vector` source and sink, and the definition can be found -[here][proto schema]. +[CUE schema][cue schema]. The protobuf schema for the `native` codec is the +same used in the `vector` source and sink, and the definition can be found in the +[proto schema][proto schema]. We will be providing more thorough guidance for using each as the feature matures. One current limitation of the JSON-based native codec is that timestamp diff --git a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md index 0d1fcc0645fcc..b7be9fad5dd50 100644 --- a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md @@ -13,8 +13,8 @@ badges: Vector's 0.22.0 release includes **breaking changes**: 1. [`gcp_stackdriver_metrics` configuration change](#stackdriver-metrics) -3. [VRL now supports template strings](#vrl-template-strings) -4. [`encode_key_value` and `encode_logfmt` quote wrapping behavior change](#encode-key-value-quote-wrapping) +2. [VRL now supports template strings](#vrl-template-strings) +3. [`encode_key_value` and `encode_logfmt` quote wrapping behavior change](#encode-key-value-quote-wrapping) and **deprecations**: @@ -68,7 +68,7 @@ project_id = "vector-123456" zone = "us-central1-a" ``` -For more information on the new syntax, you can review the documentation [here](https://vector.dev/docs/reference/configuration/sinks/gcp_stackdriver_metrics/) +For more information on the new syntax, you can review the [GCP Stackdriver Metrics sink documentation](https://vector.dev/docs/reference/configuration/sinks/gcp_stackdriver_metrics/) #### VRL now supports template strings {#vrl-template-strings} diff --git a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md index b017db9ccda3c..8464524b01c45 100644 --- a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md @@ -21,8 +21,8 @@ Vector's 0.25.0 release includes **breaking changes**: and **deprecations**: 1. [Deprecation of VRL metadata functions](#metadata-function-deprecation) -1. [Deprecation of `endpoint` option in Elasticsearch sink](#elasticsearch-endpoint-deprecation) -1. [Deprecation of the Lua version 1 API](#lua-v1-api-deprecation) +2. [Deprecation of `endpoint` option in Elasticsearch sink](#elasticsearch-endpoint-deprecation) +3. [Deprecation of the Lua version 1 API](#lua-v1-api-deprecation) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md b/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md index 847f9e796cfb2..8d221c3062213 100644 --- a/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md @@ -15,7 +15,7 @@ Vector's 0.28.0 release includes **breaking changes**: 2. [Removal of metadata functions](#metadata-functions-removal) 3. [Removal of the `apex` sink](#apex-removal) 4. [Removal of the `disk_v1` buffer type](#disk_v1-removal) -4. [Renaming of `host` to `hostname` in `datadog_logs`](#host-rename-dd-logs) +5. [Renaming of `host` to `hostname` in `datadog_logs`](#host-rename-dd-logs) and **deprecations**: diff --git a/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md b/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md index cc9b211403d01..116e1a96027c4 100644 --- a/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md @@ -12,7 +12,7 @@ badges: Vector's 0.37.0 release includes **breaking changes**: 1. [Vector defaults to requiring non-optional environment variables](#strict-env-vars) -1. [The `dnstap` source now requires the `mode` parameter](#dnstap-mode) +2. [The `dnstap` source now requires the `mode` parameter](#dnstap-mode) and **potentially impactful changes**: diff --git a/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md b/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md index 7214b6d8a7612..9f4f2976f261e 100644 --- a/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md @@ -12,7 +12,7 @@ badges: Vector's 0.31.0 release includes **breaking changes**: 1. [Removal of various deprecated internal metrics](#deprecated-internal-metrics) -1. [`component_received_event_bytes_total` and `component_sent_event_bytes_total` consistently use estimated JSON size of the event](#event_json_size) +2. [`component_received_event_bytes_total` and `component_sent_event_bytes_total` consistently use estimated JSON size of the event](#event_json_size) and **potentially impactful changes**: diff --git a/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md b/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md index a081b80ee22b7..c965c41aae404 100644 --- a/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md @@ -12,18 +12,18 @@ badges: Vector's 0.33.0 release includes **breaking changes**: 1. [Behavior of the `datadog_logs` sink's `endpoint` setting](#datadog-logs-endpoint) -1. [Disable OpenSSL legacy provider by default](#openssl-legacy-provider) +2. [Disable OpenSSL legacy provider by default](#openssl-legacy-provider) and **deprecations**: 1. [Default config location change](#default-config-location-change) -1. [Renaming the `armv7` rpm package](#armv7-rename) -1. [Metadata field in the Vector protobuf definition](#vector-proto-metadata) +2. [Renaming the `armv7` rpm package](#armv7-rename) +3. [Metadata field in the Vector protobuf definition](#vector-proto-metadata) and **potentially impactful changes**: 1. [Async runtime default number of worker threads](#runtime-worker-threads) -1. [Setting `request.concurrency = "none"`](#request-concurrency) +2. [Setting `request.concurrency = "none"`](#request-concurrency) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md b/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md index abd78eb132d99..0eb2e166d664d 100644 --- a/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md @@ -12,18 +12,18 @@ badges: Vector's 0.34.0 release includes **breaking changes**: 1. [Removal of Deprecated Datadog Component Config Options](#datadog-deprecated-config-options) -1. [Removal of Deprecated `component_name` Metric Tag](#deprecated-component-name) -1. [Removal of Deprecated Metrics Replaced by `component_errors_total`](#deprecated-component-errors-total-metrics) -1. [Removal of `peer_addr` Metric Tag](#remove-peer-addr) -1. [Blackhole sink no longer reports by default](#blackhole-sink-reporting) -1. [Remove direct OpenSSL legacy provider support](#openssl-legacy-provider) -1. [Removal of the deprecated `armv7` rpm package](#armv7-rename) -1. [Default config location change](#default-config-location-change) +2. [Removal of Deprecated `component_name` Metric Tag](#deprecated-component-name) +3. [Removal of Deprecated Metrics Replaced by `component_errors_total`](#deprecated-component-errors-total-metrics) +4. [Removal of `peer_addr` Metric Tag](#remove-peer-addr) +5. [Blackhole sink no longer reports by default](#blackhole-sink-reporting) +6. [Remove direct OpenSSL legacy provider support](#openssl-legacy-provider) +7. [Removal of the deprecated `armv7` rpm package](#armv7-rename) +8. [Default config location change](#default-config-location-change) and **deprecations**: 1. [Deprecation of `requests_completed_total`, `request_duration_seconds`, and `requests_received_total` Internal Metrics](#deprecate-obsolete-http-metrics) -1. [Deprecation of `repositories.timber.io` package repositories](#timber-package-deprecation) +2. [Deprecation of `repositories.timber.io` package repositories](#timber-package-deprecation) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2023-11-07-new-linux-repos.md b/website/content/en/highlights/2023-11-07-new-linux-repos.md index 83ad54738fca4..b437833a51a8d 100644 --- a/website/content/en/highlights/2023-11-07-new-linux-repos.md +++ b/website/content/en/highlights/2023-11-07-new-linux-repos.md @@ -54,14 +54,14 @@ existing repository to your discretion. rm "/etc/apt/sources.list.d/timber-vector.list" ``` -2. Run the following commands to set up APT to download through HTTPS: +1. Run the following commands to set up APT to download through HTTPS: ```sh sudo apt-get update sudo apt-get install apt-transport-https curl gnupg ``` -3. Run the following commands to set up the Vector `deb` repo on your system +1. Run the following commands to set up the Vector `deb` repo on your system and create a Datadog archive keyring: ```sh @@ -73,7 +73,7 @@ curl https://keys.datadoghq.com/DATADOG_APT_KEY_F14F620E.public | sudo gpg --no- curl https://keys.datadoghq.com/DATADOG_APT_KEY_C0962C7D.public | sudo gpg --no-default-keyring --keyring /usr/share/keyrings/datadog-archive-keyring.gpg --import --batch ``` -4. Run the following commands to update your local `apt` repo and install Vector: +1. Run the following commands to update your local `apt` repo and install Vector: ```sh sudo apt-get update @@ -91,7 +91,7 @@ sudo apt-get install vector rm "/etc/yum.repos.d/timber-vector.repo" ``` -2. Run the following commands to set up the Vector `rpm` repo on your system: +1. Run the following commands to set up the Vector `rpm` repo on your system: ```sh cat < /etc/yum.repos.d/vector.repo @@ -109,7 +109,7 @@ EOF **Note:** If you are running RHEL 8.1 or CentOS 8.1, use `repo_gpgcheck=0` instead of `repo_gpgcheck=1` in the configuration above. -3. Update your packages and install Vector: +1. Update your packages and install Vector: ```sh sudo yum makecache diff --git a/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md b/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md index de9408043cac2..f4c5081b7f9e8 100644 --- a/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md @@ -12,9 +12,9 @@ badges: Vector's 0.35.0 release includes **breaking changes**: 1. [The Throttle transform's `events_discarded_total` internal metric is now opt-in](#events-discarded-total-opt-in) -1. [The `file` internal metric tag is now opt-in for file-based components](#file-tag-opt-in) -1. [Datadog sinks now default the API key and Site to the value of environment variables](#datadog-env-vars) -1. [Removal of `requests_completed_total`, `request_duration_seconds`, and `requests_received_total` Internal Metrics](#remove-obsolete-http-metrics) +2. [The `file` internal metric tag is now opt-in for file-based components](#file-tag-opt-in) +3. [Datadog sinks now default the API key and Site to the value of environment variables](#datadog-env-vars) +4. [Removal of `requests_completed_total`, `request_duration_seconds`, and `requests_received_total` Internal Metrics](#remove-obsolete-http-metrics) and **deprecations**: @@ -23,9 +23,9 @@ and **deprecations**: and **potentially impactful changes**: 1. [HTTP server-based sources now include a `keepalive.max_connection_age_secs` config option that defaults to 5 minutes](#http-keepalive-max-connection-age) -1. [Update `component_sent_bytes_total` to correctly report uncompressed bytes for all sinks](#component-sent-bytes-total) -1. [Update `component_received_bytes_total` to correctly report decompressed bytes for all sources](#component-received-bytes-total) -1. [Update default values for the `request.retry_max_duration_secs` and `request.rate_limit_num` sink configuration options](#request-config-options) +2. [Update `component_sent_bytes_total` to correctly report uncompressed bytes for all sinks](#component-sent-bytes-total) +3. [Update `component_received_bytes_total` to correctly report decompressed bytes for all sources](#component-received-bytes-total) +4. [Update default values for the `request.retry_max_duration_secs` and `request.rate_limit_num` sink configuration options](#request-config-options) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md b/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md index 90a7239d71ebf..379b7ad197ee2 100644 --- a/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md @@ -16,7 +16,7 @@ Vector's 0.38.0 release includes a **breaking change**: and **deprecations**: 1. [Deprecation of the path coalescing operator](#path-coalescing) -1. [Deprecation of `enterprise` configuration](#enterprise-configuration) +2. [Deprecation of `enterprise` configuration](#enterprise-configuration) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md b/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md index 784d2602f2592..dae63b3e6da72 100644 --- a/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md @@ -12,7 +12,7 @@ badges: Vector's 0.39.0 release includes two **breaking changes**: 1. [Removal of the path coalescing operator](#path-coalescing) -1. [Removal of `enterprise` configuration](#enterprise-configuration) +2. [Removal of `enterprise` configuration](#enterprise-configuration) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md index a2f604eeaf66f..5813dfc248a40 100644 --- a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md @@ -12,8 +12,8 @@ badges: Vector's 0.40.0 release includes three **breaking changes**: 1. [GELF codec defaults to null-delimited messages for stream-based sources](#gelf) -1. [Reduce transforms now properly aggregate nested fields](#reduce-aggregate) -1. [Vector no longer supports CentOS 7](#centos7) +2. [Reduce transforms now properly aggregate nested fields](#reduce-aggregate) +3. [Vector no longer supports CentOS 7](#centos7) We cover them below to help you upgrade quickly: From 7dc3ce3f34de6b79c0f09b4e44fe0e9d87cc09c2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 25 Mar 2026 18:32:34 -0400 Subject: [PATCH 007/364] chore(dev): add links to design issues (#25046) * chore: add TODO comments linking to log_namespace design issues https://github.com/vectordotdev/vector/issues/25044 https://github.com/vectordotdev/vector/issues/25045 Co-Authored-By: Claude Sonnet 4.6 (1M context) * docs fix --------- Co-authored-by: Claude Sonnet 4.6 (1M context) --- lib/codecs/src/decoding/format/mod.rs | 2 ++ lib/codecs/src/decoding/format/otlp.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 701d11c8403d5..cbb2172c3392d 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -50,6 +50,8 @@ pub trait Deserializer: DynClone + Send + Sync { /// by not requiring heap allocations for it. /// /// **Note**: The type of the produced events depends on the implementation. + /// + /// TODO: fn parse( &self, bytes: Bytes, diff --git a/lib/codecs/src/decoding/format/otlp.rs b/lib/codecs/src/decoding/format/otlp.rs index 2004df92269ca..29b3477e4398f 100644 --- a/lib/codecs/src/decoding/format/otlp.rs +++ b/lib/codecs/src/decoding/format/otlp.rs @@ -181,6 +181,7 @@ impl Deserializer for OtlpDeserializer { } } OtlpSignalType::Traces => { + // TODO: if let Ok(mut events) = self.traces_deserializer.parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() From 4fba57e9c4d6561c126851df8c4065221b44e992 Mon Sep 17 00:00:00 2001 From: Oleksandr Zanichkovskyi Date: Wed, 25 Mar 2026 23:51:02 +0100 Subject: [PATCH 008/364] feat(opentelemetry source): allow to add headers to metadata for OpenTelemetry metrics and traces (#24942) * feat: allow to add headers to metadata for OpenTelemetry metrics and traces * chore: code formatting * chore: refactored the implementation and tests * chore: added changelog * chore: fixed formatting * chore: updated docs * make generate-docs * feat: pass log_namespace when enriching metrics with header for use_otlp_decoding.metrics = true * chore: removed log_namespace comment related to metrics and traces * chore: fixed review comments * update comment and regen docs * chore: revert log_namespace threading in build_warp_trace_filter The semantics of log_namespace for traces have not been defined yet. The correct fix will be addressed in a dedicated follow-up branch. Co-Authored-By: Claude Sonnet 4.6 (1M context) * chore: revert log_namespace threading in build_warp_trace_filter The semantics of log_namespace for traces have not been defined yet. The correct fix will be addressed in a dedicated follow-up branch. Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Sonnet 4.6 (1M context) --- ...telemetry-header-enrichment.enhancement.md | 7 + src/sources/opentelemetry/config.rs | 7 +- src/sources/opentelemetry/http.rs | 25 +- src/sources/opentelemetry/tests.rs | 288 +++++++++++++++++- src/sources/util/http/headers.rs | 176 +++++++++-- .../sources/generated/opentelemetry.cue | 7 +- 6 files changed, 463 insertions(+), 47 deletions(-) create mode 100644 changelog.d/24619-opentelemetry-header-enrichment.enhancement.md diff --git a/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md b/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md new file mode 100644 index 0000000000000..e0b0b33a5caa3 --- /dev/null +++ b/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md @@ -0,0 +1,7 @@ +`opentelemetry` source: Implemented header enrichment for OTLP metrics and traces. Unlike logs, which support enriching +the event itself or its metadata, depending on `log_namespace` settings, for metrics and traces this setting is ignored +and header values are added to the event metadata. + +Issue: https://github.com/vectordotdev/vector/issues/24619 + +authors: ozanichkovsky diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index c7eed8f301b50..89f6163da0595 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -202,13 +202,14 @@ pub struct HttpConfig { #[serde(default)] pub keepalive: KeepaliveConfig, - /// A list of HTTP headers to include in the log event. + /// A list of HTTP headers to include in the event. /// /// Accepts the wildcard (`*`) character for headers matching a specified pattern. /// - /// Specifying "*" results in all headers included in the log event. + /// Specifying "*" results in all headers included in the event. /// - /// These headers are not included in the JSON payload if a field with a conflicting name exists. + /// For log events in legacy namespace mode, headers are not included if a field with a conflicting name exists. + /// For metrics and traces, headers are always added to event metadata. #[serde(default)] #[configurable(metadata(docs::examples = "User-Agent"))] #[configurable(metadata(docs::examples = "X-My-Custom-Header"))] diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index f15e44bbd0926..42be681f1cb90 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -109,9 +109,11 @@ pub(crate) fn build_warp_filter( ); let metrics_filters = build_warp_metrics_filter( acknowledgements, + log_namespace, out.clone(), bytes_received.clone(), events_received.clone(), + headers.clone(), metrics_deserializer, ); let trace_filters = build_warp_trace_filter( @@ -119,6 +121,7 @@ pub(crate) fn build_warp_filter( out.clone(), bytes_received, events_received, + headers.clone(), traces_deserializer, ); log_filters @@ -257,12 +260,14 @@ fn build_warp_log_filter( } fn build_warp_metrics_filter( acknowledgements: bool, + log_namespace: LogNamespace, source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + headers_cfg: Vec, deserializer: Option, ) -> BoxedFilter<(Response,)> { - let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { decompress_body(encoding_header.as_deref(), body) .inspect_err(|err| { // Other status codes are already handled by `sources::util::decompress_body` (tech debt). @@ -276,15 +281,14 @@ fn build_warp_metrics_filter( .and_then(|decoded_body| { bytes_received.emit(ByteSize(decoded_body.len())); if let Some(d) = deserializer.as_ref() { - parse_with_deserializer( - d, - decoded_body, - LogNamespace::default(), - &events_received, - ) + parse_with_deserializer(d, decoded_body, log_namespace, &events_received) } else { decode_metrics_body(decoded_body, &events_received) } + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, log_namespace); + events + }) }) }; @@ -301,9 +305,10 @@ fn build_warp_trace_filter( source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + headers_cfg: Vec, deserializer: Option, ) -> BoxedFilter<(Response,)> { - let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { decompress_body(encoding_header.as_deref(), body) .inspect_err(|err| { // Other status codes are already handled by `sources::util::decompress_body` (tech debt). @@ -326,6 +331,10 @@ fn build_warp_trace_filter( } else { decode_trace_body(decoded_body, &events_received) } + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, LogNamespace::default()); + events + }) }) }; diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index de5781756dba1..9fb0bac71a1cd 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -4,12 +4,31 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +use crate::{ + SourceSender, + config::{OutputId, SourceConfig, SourceContext}, + event::{ + Event, EventStatus, LogEvent, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, + ObjectMap, Value, into_event_stream, + metric::{Bucket, Quantile}, + }, + sources::opentelemetry::config::{ + GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, + }, + test_util::{ + self, + addr::next_addr, + components::{SOURCE_TAGS, assert_source_compliance}, + }, +}; use chrono::{DateTime, TimeZone, Utc}; use futures::Stream; use futures_util::StreamExt; use prost::Message; use similar_asserts::assert_eq; use tonic::Request; +use vector_lib::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; +use vector_lib::opentelemetry::proto::trace::v1::{ResourceSpans, ScopeSpans, Span}; use vector_lib::{ config::LogNamespace, lookup::path, @@ -33,22 +52,6 @@ use vector_lib::{ }; use vrl::value; -use crate::{ - SourceSender, - config::{OutputId, SourceConfig, SourceContext}, - event::{ - Event, EventStatus, LogEvent, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, - ObjectMap, Value, into_event_stream, - metric::{Bucket, Quantile}, - }, - sources::opentelemetry::config::{GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig}, - test_util::{ - self, - addr::next_addr, - components::{SOURCE_TAGS, assert_source_compliance}, - }, -}; - fn create_test_logs_request() -> Request { Request::new(ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { @@ -100,6 +103,104 @@ fn create_test_logs_request() -> Request { }) } +fn create_test_metrics_request() -> ExportMetricsServiceRequest { + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(StringValue("vector-collector".to_string())), + }), + }], + dropped_attributes_count: 0, + }), + schema_url: "".to_string(), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { + name: "vector-collector-instrumentation".to_string(), + version: "0.111.0".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), + schema_url: "".to_string(), + metrics: vec![Metric { + name: "some.random.metric".to_string(), + description: "Some random metric we use for test".to_string(), + unit: "1".to_string(), + data: Some(Data::Summary(Summary { + data_points: vec![SummaryDataPoint { + attributes: vec![ + KeyValue { + key: "host".to_string(), + value: Some(AnyValue { + value: Some(StringValue("localhost".to_string())), + }), + }, + KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(StringValue("vector-collector".to_string())), + }), + }, + ], + start_time_unix_nano: 0, + time_unix_nano: 0, + count: 5, + sum: 122.5, + quantile_values: vec![ + ValueAtQuantile { + quantile: 0.5, + value: 24.5, + }, + ValueAtQuantile { + quantile: 0.9, + value: 45.0, + }, + ValueAtQuantile { + quantile: 1.0, + value: 60.0, + }, + ], + flags: 0, + }], + })), + }], + }], + }], + } +} + +fn create_test_traces_request() -> ExportTraceServiceRequest { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: None, + scope_spans: vec![ScopeSpans { + scope: None, + spans: vec![Span { + trace_id: (1..17).collect::>(), + span_id: (1..9).collect::>(), + parent_span_id: (1..9).collect::>(), + name: "span".to_string(), + kind: 1, + start_time_unix_nano: 1713525203000000000, + end_time_unix_nano: 1713525205000000000, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + trace_state: "".to_string(), + }], + schema_url: "".to_string(), + }], + schema_url: "".to_string(), + }], + } +} + #[test] fn generate_config() { test_util::test_generate_config::(); @@ -1091,6 +1192,39 @@ fn get_source_config_with_headers( } } +async fn send_and_collect_otel_event( + use_otlp_decoding: bool, + output_port: &str, + endpoint: &str, + body: Vec, +) -> Event { + let (_guard_0, grpc_addr) = next_addr(); + let (_guard_1, http_addr) = next_addr(); + + let source = get_source_config_with_headers(grpc_addr, http_addr, use_otlp_decoding); + + let (sender, output, _) = new_source(EventStatus::Delivered, output_port.to_string()); + let server = source + .build(SourceContext::new_test(sender, None)) + .await + .unwrap(); + tokio::spawn(server); + test_util::wait_for_tcp(http_addr).await; + + let _res = reqwest::Client::new() + .post(format!("http://{http_addr}/{endpoint}")) + .header("Content-Type", "application/x-protobuf") + .header("User-Agent", "Test") + .body(body) + .send() + .await + .expect("Failed to send request to OpenTelemetry source."); + + let mut events = test_util::collect_ready(output).await; + assert_eq!(events.len(), 1); + events.pop().unwrap() +} + #[tokio::test] async fn http_headers_logs_use_otlp_decoding_false() { assert_source_compliance(&SOURCE_TAGS, async { @@ -1237,6 +1371,128 @@ async fn http_headers_logs_use_otlp_decoding_true() { .await; } +#[tokio::test] +async fn http_headers_metrics_use_otlp_decoding_false() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + false, + METRICS, + "v1/metrics", + create_test_metrics_request().encode_to_vec(), + ) + .await; + let metric = event.as_metric(); + assert_eq!( + metric + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + metric + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + +#[tokio::test] +async fn http_headers_metrics_use_otlp_decoding_true() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + true, + METRICS, + "v1/metrics", + create_test_metrics_request().encode_to_vec(), + ) + .await; + let log = event.as_log(); + assert_eq!(log["AbsentHeader"], Value::Null); + assert_eq!(log["User-Agent"], "Test".into()); + }) + .await; +} + +#[tokio::test] +async fn http_headers_traces_use_otlp_decoding_false() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + false, + TRACES, + "v1/traces", + create_test_traces_request().encode_to_vec(), + ) + .await; + let trace = event.as_trace(); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + +#[tokio::test] +async fn http_headers_traces_use_otlp_decoding_true() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + true, + TRACES, + "v1/traces", + create_test_traces_request().encode_to_vec(), + ) + .await; + let trace = event.as_trace(); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + pub struct OTelTestEnv { pub grpc_addr: String, pub config: OpentelemetryConfig, diff --git a/src/sources/util/http/headers.rs b/src/sources/util/http/headers.rs index 8a4b73ff212df..ef8376f695723 100644 --- a/src/sources/util/http/headers.rs +++ b/src/sources/util/http/headers.rs @@ -24,14 +24,22 @@ pub fn add_headers( let value = headers.get(header_name).map(HeaderValue::as_bytes); for event in events.iter_mut() { - if let Event::Log(log) = event { - log_namespace.insert_source_metadata( - source_name, - log, - Some(LegacyKey::InsertIfEmpty(path!(header_name))), - path!("headers", header_name), - Value::from(value.map(Bytes::copy_from_slice)), - ); + match event { + Event::Log(log) => { + log_namespace.insert_source_metadata( + source_name, + log, + Some(LegacyKey::InsertIfEmpty(path!(header_name))), + path!("headers", header_name), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } + Event::Metric(_) | Event::Trace(_) => { + event.metadata_mut().value_mut().insert( + path!(source_name, "headers", header_name), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } } } } @@ -45,14 +53,22 @@ pub fn add_headers( let value = headers.get(header_name).map(HeaderValue::as_bytes); for event in events.iter_mut() { - if let Event::Log(log) = event { - log_namespace.insert_source_metadata( - source_name, - log, - Some(LegacyKey::InsertIfEmpty(path!(header_name.as_str()))), - path!("headers", header_name.as_str()), - Value::from(value.map(Bytes::copy_from_slice)), - ); + match event { + Event::Log(log) => { + log_namespace.insert_source_metadata( + source_name, + log, + Some(LegacyKey::InsertIfEmpty(path!(header_name.as_str()))), + path!("headers", header_name.as_str()), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } + Event::Metric(_) | Event::Trace(_) => { + event.metadata_mut().value_mut().insert( + path!(source_name, "headers", header_name.as_str()), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } } } } @@ -64,12 +80,15 @@ pub fn add_headers( #[cfg(test)] mod tests { + use chrono::{DateTime, Utc}; + use std::time::SystemTime; use vector_lib::config::LogNamespace; use vrl::{path, value}; use warp::http::HeaderMap; + use crate::event::{Event, MetricKind, MetricTags, MetricValue}; use crate::{ - event::LogEvent, + event::{LogEvent, Metric, TraceEvent}, sources::{http_server::HttpConfigParamKind, util::add_headers}, }; @@ -109,6 +128,53 @@ mod tests { .get(path!("test", "headers")) .unwrap() ); + + let mut metric = [Event::from( + Metric::new( + "some.random.metric", + MetricKind::Incremental, + MetricValue::Counter { value: 123.4 }, + ) + .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_tags(Some(MetricTags::default())), + )]; + + add_headers( + &mut metric, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let mut trace = [TraceEvent::from(btreemap! { + "span_id" => "abc123", + "trace_id" => "xyz789", + "span_name" => "test-span", + "service" => "my-service", + }) + .into()]; + + add_headers( + &mut trace, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + assert_eq!( + metric[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(), + trace[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap() + ); } #[test] @@ -162,5 +228,81 @@ mod tests { "gzip".into(), "Checking log contains Content-Encoding header" ); + + let mut metric = [Event::from( + Metric::new( + "some.random.metric", + MetricKind::Incremental, + MetricValue::Counter { value: 123.4 }, + ) + .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_tags(Some(MetricTags::default())), + )]; + + add_headers( + &mut metric, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let metric_headers = metric[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(); + + assert_eq!( + metric_headers.get("content-type").unwrap(), + &value!("application/x-protobuf"), + "Checking metric contains Content-Type header" + ); + assert!( + !metric_headers.contains("user-agent"), + "Checking metric does not contain User-Agent header" + ); + assert_eq!( + metric_headers.get("content-encoding").unwrap(), + &value!("gzip"), + "Checking metric contains Content-Encoding header" + ); + + let mut trace = [TraceEvent::from(btreemap! { + "span_id" => "abc123", + "trace_id" => "xyz789", + "span_name" => "test-span", + "service" => "my-service", + }) + .into()]; + + add_headers( + &mut trace, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let trace_headers = trace[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(); + + assert_eq!( + trace_headers.get("content-type").unwrap(), + &value!("application/x-protobuf"), + "Checking trace contains Content-Type header" + ); + assert!( + !trace_headers.contains("user-agent"), + "Checking trace does not contain User-Agent header" + ); + assert_eq!( + trace_headers.get("content-encoding").unwrap(), + &value!("gzip"), + "Checking trace contains Content-Encoding header" + ); } } diff --git a/website/cue/reference/components/sources/generated/opentelemetry.cue b/website/cue/reference/components/sources/generated/opentelemetry.cue index d3b498506a323..ca4150550c771 100644 --- a/website/cue/reference/components/sources/generated/opentelemetry.cue +++ b/website/cue/reference/components/sources/generated/opentelemetry.cue @@ -170,13 +170,14 @@ generated: components: sources: opentelemetry: configuration: { } headers: { description: """ - A list of HTTP headers to include in the log event. + A list of HTTP headers to include in the event. Accepts the wildcard (`*`) character for headers matching a specified pattern. - Specifying "*" results in all headers included in the log event. + Specifying "*" results in all headers included in the event. - These headers are not included in the JSON payload if a field with a conflicting name exists. + For log events in legacy namespace mode, headers are not included if a field with a conflicting name exists. + For metrics and traces, headers are always added to event metadata. """ required: false type: array: { From 498d1daa38432a329d168a2eec97a212922e71bc Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 25 Mar 2026 20:51:47 -0400 Subject: [PATCH 009/364] chore(ci): improve CLA instructions (#25039) * chore(cla): improve bot message with copy-pastable sign instruction Add a clearer `custom-notsigned-prcomment` that includes the sign phrase in a code block (one-click copy in GitHub UI) and a note for contributors whose commit email is not linked to their GitHub account. Co-Authored-By: Claude Sonnet 4.6 * repurpose --------- Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/cla.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 2f48f1f1d1edf..536e364a402b9 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -14,9 +14,9 @@ jobs: CLAAssistant: runs-on: ubuntu-latest permissions: - pull-requests: write # Required to comment on PRs - id-token: write # Required to federate tokens with dd-octo-sts-action - actions: write # Required to create/update workflow runs + pull-requests: write # Required to comment on PRs + id-token: write # Required to federate tokens with dd-octo-sts-action + actions: write # Required to create/update workflow runs steps: - name: CLA already verified on PR if: github.event_name == 'merge_group' @@ -36,17 +36,16 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ steps.octo-sts.outputs.token }} with: - path-to-signatures: 'cla.json' - path-to-document: 'https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618' # e.g. a CLA or a DCO document - branch: 'vector' + path-to-signatures: "cla.json" + path-to-document: "https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618" # e.g. a CLA or a DCO document + branch: "vector" remote-repository-name: cla-signatures remote-organization-name: DataDog allowlist: step-security-bot - # the followings are the optional inputs - If the optional inputs are not given, then default values will be taken - #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' - #signed-commit-message: 'For example: $contributorName has signed the CLA in $owner/$repo#$pullRequestNo' - #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' - #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' - #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' - #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) + custom-notsigned-prcomment: | + Thank you for your contribution! Before we can merge this PR, please sign our [Contributor License Agreement](https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618). + + To sign, copy and post the phrase below as a new comment on this PR. + + > **Note:** If the bot says your username was not found, the email used in your git commit may not be linked to your GitHub account. Fix this at [github.com/settings/emails](https://github.com/settings/emails), then comment `recheck` to retry. From bc1e6bdde6b07f51efd4764c65d75c32d056499c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:10:51 +0000 Subject: [PATCH 010/364] chore(deps): bump uuid from 1.18.1 to 1.22.0 (#25025) * chore(deps): bump uuid from 1.18.1 to 1.22.0 Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.18.1 to 1.22.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.18.1...v1.22.0) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.22.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 49 +++++++++++++++++++++++++++++++------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 + 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c5efc9802ce97..4a6ddc9d8b9e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,7 +32,7 @@ checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.11", ] [[package]] @@ -2215,7 +2215,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.11", +] + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", ] [[package]] @@ -2225,7 +2236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -2768,6 +2779,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.3.0" @@ -3019,7 +3039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -5797,7 +5817,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.11", ] [[package]] @@ -7815,7 +7835,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.11", "opaque-debug", "universal-hash", ] @@ -8592,6 +8612,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ + "chacha20 0.10.0", "getrandom 0.4.2", "rand_core 0.10.0", ] @@ -9954,7 +9975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "digest", ] @@ -9965,7 +9986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "digest", ] @@ -9976,7 +9997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "digest", ] @@ -12092,14 +12113,14 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.18.1" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.2", "js-sys", - "rand 0.9.2", - "serde", + "rand 0.10.0", + "serde_core", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 06b83efedb33e..5dc1dda8072da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,7 @@ tonic-reflection = { version = "0.11", default-features = false, features = ["se tracing = { version = "0.1.44", default-features = false } tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } url = { version = "2.5.4", default-features = false, features = ["serde"] } -uuid = { version = "1.18.1", features = ["v4", "v7", "serde", "fast-rng"] } +uuid = { version = "1.22.0", features = ["v4", "v7", "serde", "fast-rng"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8a5759685500e..8dbc10abc4f7e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -144,6 +144,7 @@ cesu8,https://github.com/emk/cesu8-rs,Apache-2.0 OR MIT,Eric Kidd chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers +chacha20,https://github.com/RustCrypto/stream-ciphers,MIT OR Apache-2.0,RustCrypto Developers chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors From 072f6284c83279cd920177b32114f10f8fd5164c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:15:29 +0000 Subject: [PATCH 011/364] chore(deps): bump snafu from 0.8.9 to 0.9.0 (#25029) Bumps [snafu](https://github.com/shepmaster/snafu) from 0.8.9 to 0.9.0. - [Changelog](https://github.com/shepmaster/snafu/blob/main/CHANGELOG.md) - [Commits](https://github.com/shepmaster/snafu/compare/0.8.9...0.9.0) --- updated-dependencies: - dependency-name: snafu dependency-version: 0.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 45 +++++++++++++++++++++++++++++++++------------ Cargo.toml | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a6ddc9d8b9e6..05055530add86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2455,7 +2455,7 @@ dependencies = [ "serde_with", "similar-asserts", "smallvec", - "snafu 0.8.9", + "snafu 0.9.0", "strum 0.28.0", "syslog_loose 0.23.0", "tokio", @@ -3435,7 +3435,7 @@ dependencies = [ "criterion", "data-encoding", "hickory-proto 0.25.2", - "snafu 0.8.9", + "snafu 0.9.0", ] [[package]] @@ -3452,7 +3452,7 @@ dependencies = [ "pastey", "prost 0.12.6", "prost-build 0.12.6", - "snafu 0.8.9", + "snafu 0.9.0", "tracing 0.1.44", "vector-common", "vector-config", @@ -3474,7 +3474,7 @@ dependencies = [ "anyhow", "serde", "serde_json", - "snafu 0.8.9", + "snafu 0.9.0", "tracing 0.1.44", "vector-config", "vector-config-common", @@ -3681,7 +3681,7 @@ dependencies = [ "const-str", "dyn-clone", "indoc", - "snafu 0.8.9", + "snafu 0.9.0", "vector-vrl-category", "vrl", ] @@ -8096,7 +8096,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.8.9", + "snafu 0.9.0", "vector-common", ] @@ -10210,10 +10210,19 @@ name = "snafu" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive 0.8.9", +] + +[[package]] +name = "snafu" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" dependencies = [ "futures-core", "pin-project", - "snafu-derive 0.8.9", + "snafu-derive 0.9.0", ] [[package]] @@ -10240,6 +10249,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "snafu-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +dependencies = [ + "heck 0.5.0", + "proc-macro2 1.0.106", + "quote 1.0.44", + "syn 2.0.117", +] + [[package]] name = "snap" version = "1.1.1" @@ -12342,7 +12363,7 @@ dependencies = [ "similar-asserts", "smallvec", "smpl_jwt", - "snafu 0.8.9", + "snafu 0.9.0", "snap", "socket2 0.5.10", "sqlx", @@ -12402,7 +12423,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.8.9", + "snafu 0.9.0", "tokio", "tokio-stream", "tonic 0.11.0", @@ -12441,7 +12462,7 @@ dependencies = [ "rkyv", "serde", "serde_yaml", - "snafu 0.8.9", + "snafu 0.9.0", "temp-dir", "tokio", "tokio-test", @@ -12504,7 +12525,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "snafu 0.8.9", + "snafu 0.9.0", "toml", "tracing 0.1.44", "url", @@ -12598,7 +12619,7 @@ dependencies = [ "serde_yaml", "similar-asserts", "smallvec", - "snafu 0.8.9", + "snafu 0.9.0", "socket2 0.5.10", "tokio", "tokio-openssl", diff --git a/Cargo.toml b/Cargo.toml index 5dc1dda8072da..42e2b2c24a12f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,7 @@ semver = { version = "1.0.27", default-features = false, features = ["serde", "s serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } serde_yaml = { version = "0.9.34", default-features = false } -snafu = { version = "0.8.9", default-features = false, features = ["futures", "std"] } +snafu = { version = "0.9.0", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } tempfile = "3.23.0" tokio = { version = "1.49.0", default-features = false } From 041769359ba0a9d68017e10e6e84cb7a83b3df27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:22:32 +0000 Subject: [PATCH 012/364] chore(website deps): bump picomatch from 2.3.1 to 2.3.2 in /website (#25047) Bumps [picomatch](https://github.com/micromatch/picomatch) from 2.3.1 to 2.3.2. - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/2.3.1...2.3.2) --- updated-dependencies: - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 57cd6d50741c0..95702b3c49289 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -3055,9 +3055,9 @@ picocolors@^1.0.0, picocolors@^1.1.1: integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: - version "2.3.1" - resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" - integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + version "2.3.2" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.2.tgz#5a942915e26b372dc0f0e6753149a16e6b1c5601" + integrity sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA== pify@^2.3.0: version "2.3.0" From d0c6cea89661577079d3eec790e4c054772de7d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:25:49 +0000 Subject: [PATCH 013/364] chore(website deps): bump yaml from 1.10.2 to 1.10.3 in /website (#25043) Bumps [yaml](https://github.com/eemeli/yaml) from 1.10.2 to 1.10.3. - [Release notes](https://github.com/eemeli/yaml/releases) - [Commits](https://github.com/eemeli/yaml/compare/v1.10.2...v1.10.3) --- updated-dependencies: - dependency-name: yaml dependency-version: 1.10.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 95702b3c49289..f7cedc6f6748f 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -3876,9 +3876,9 @@ yallist@^3.0.2: integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== yaml@^1.10.0, yaml@^1.10.2: - version "1.10.2" - resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" - integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + version "1.10.3" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.3.tgz#76e407ed95c42684fb8e14641e5de62fe65bbcb3" + integrity sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA== yargs-parser@^20.2.2: version "20.2.9" From 5fcf1516151f659b5832fc433a4e10a219e31c7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:26:25 +0000 Subject: [PATCH 014/364] chore(deps): bump ipnet from 2.11.0 to 2.12.0 (#25030) Bumps [ipnet](https://github.com/krisprice/ipnet) from 2.11.0 to 2.12.0. - [Release notes](https://github.com/krisprice/ipnet/releases) - [Changelog](https://github.com/krisprice/ipnet/blob/master/RELEASES.md) - [Commits](https://github.com/krisprice/ipnet/compare/2.11.0...2.12.0) --- updated-dependencies: - dependency-name: ipnet dependency-version: 2.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05055530add86..76f557baf7a8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5542,9 +5542,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" dependencies = [ "serde", ] From 09629f02d60614a38b20b49cb10acbce47a9f6cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:49:33 +0000 Subject: [PATCH 015/364] chore(ci): bump docker/setup-buildx-action from 3.12.0 to 4.0.0 (#25003) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.12.0 to 4.0.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/8d2750c68a42422c14e847fe6c8ac0403b4cbd6f...4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/environment.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/regression.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index fce6b25c3c181..75680427712a5 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -43,7 +43,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 if: env.SHOULD_PUBLISH == 'true' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3c533c593aeb6..9b5385a38b39f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -347,7 +347,7 @@ jobs: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 with: version: latest - name: Download all Linux package artifacts diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index a8c6981e53a9d..75a84c0fc1afe 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -209,7 +209,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build 'vector' target image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 @@ -252,7 +252,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build 'vector' target image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 From 3255b66b2e8a2fb7ec3412904b856a8349a236ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:49:57 +0000 Subject: [PATCH 016/364] chore(ci): bump actions/cache from 5.0.3 to 5.0.4 (#25002) Bumps [actions/cache](https://github.com/actions/cache) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/cdf6c1fa76f9f475f3d7449005a359c84ca0f306...668228422ae6a00e4ad889ee87cd7109ec5666a7) --- updated-dependencies: - dependency-name: actions/cache dependency-version: 5.0.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cross.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index 8cf8706b1a42c..9519a7d79df94 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -35,7 +35,7 @@ jobs: with: ref: ${{ inputs.ref }} - - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 name: Cache Cargo registry + index with: path: | From 9337c80bb9d2491a8d617df408c20a88385d719a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:50:14 +0000 Subject: [PATCH 017/364] chore(ci): bump docker/login-action from 3.7.0 to 4.0.0 (#25001) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.7.0 to 4.0.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/c94ce9fb468520275223c153574b00df6fe4bcc9...b45d80f862d83dbcd57f89517bcf500b2ab88fb2) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-test-runner.yml | 2 +- .github/workflows/environment.yml | 2 +- .github/workflows/publish.yml | 4 ++-- .github/workflows/regression.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test-runner.yml b/.github/workflows/build-test-runner.yml index 50a46fd4bd30f..7a7596651ed2c 100644 --- a/.github/workflows/build-test-runner.yml +++ b/.github/workflows/build-test-runner.yml @@ -36,7 +36,7 @@ jobs: vdev: true - name: Login to GitHub Container Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 75680427712a5..cf6313e4ce894 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -45,7 +45,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 if: env.SHOULD_PUBLISH == 'true' with: username: ${{ secrets.CI_DOCKER_USERNAME }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9b5385a38b39f..0594151b2db23 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -331,12 +331,12 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 75a84c0fc1afe..f2216df39e684 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -326,7 +326,7 @@ jobs: uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -366,7 +366,7 @@ jobs: uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ${{ steps.login-ecr.outputs.registry }} From 84abc68df859d58b2a7681757e52aca7a8094ad2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 20:24:22 +0000 Subject: [PATCH 018/364] chore(deps): bump bollard from 0.19.2 to 0.20.2 (#25026) * chore(deps): bump bollard from 0.19.2 to 0.20.2 Bumps [bollard](https://github.com/fussybeaver/bollard) from 0.19.2 to 0.20.2. - [Release notes](https://github.com/fussybeaver/bollard/releases) - [Changelog](https://github.com/fussybeaver/bollard/blob/master/RELEASE.md) - [Commits](https://github.com/fussybeaver/bollard/compare/v0.19.2...v0.20.2) --- updated-dependencies: - dependency-name: bollard dependency-version: 0.20.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump bollard in Cargo.toml * Fix incompatible changes --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 11 ++++------- Cargo.toml | 2 +- src/docker.rs | 5 ++--- src/sources/docker_logs/tests.rs | 4 ++-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76f557baf7a8a..be47e7ba18697 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1882,9 +1882,9 @@ dependencies = [ [[package]] name = "bollard" -version = "0.19.2" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8796b390a5b4c86f9f2e8173a68c2791f4fa6b038b84e96dbc01c016d1e6722c" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "base64 0.22.1", "bollard-stubs", @@ -1905,12 +1905,10 @@ dependencies = [ "pin-project-lite", "rustls 0.23.23", "rustls-native-certs 0.8.1", - "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", "serde_derive", "serde_json", - "serde_repr", "serde_urlencoded", "thiserror 2.0.17", "tokio", @@ -1922,15 +1920,14 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.49.0-rc.28.3.3" +version = "1.52.1-rc.29.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7814991259013d5a5bee4ae28657dae0747d843cf06c40f7fc0c2894d6fa38" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" dependencies = [ "chrono", "serde", "serde_json", "serde_repr", - "serde_with", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 42e2b2c24a12f..a139a477d5ab2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -354,7 +354,7 @@ arrow-schema = { version = "56.2.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } base64 = { workspace = true, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } -bollard = { version = "0.19.1", default-features = false, features = ["pipe", "ssl", "chrono"], optional = true } +bollard = { version = "0.20", default-features = false, features = ["pipe", "ssl", "chrono"], optional = true } bytes = { workspace = true, features = ["serde"] } bytesize = { version = "2.0.1", default-features = false } chrono.workspace = true diff --git a/src/docker.rs b/src/docker.rs index cd13a4c4c54ce..381ffddac0219 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -4,12 +4,11 @@ use std::{collections::HashMap, env, path::PathBuf}; use bollard::{ API_DEFAULT_VERSION, Docker, errors::Error as DockerError, - models::HostConfig, + models::{ContainerCreateBody, HostConfig}, query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptionsBuilder, ListImagesOptionsBuilder, RemoveContainerOptions, StartContainerOptions, StopContainerOptions, }, - secret::ContainerCreateBody, }; use futures::StreamExt; use http::uri::Uri; @@ -130,7 +129,7 @@ async fn pull_image(docker: &Docker, image: &str, tag: &str) { .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); - if let Some(error) = info.error { + if let Some(error) = info.error_detail { panic!("{error:?}"); } }) diff --git a/src/sources/docker_logs/tests.rs b/src/sources/docker_logs/tests.rs index f5af697c1b9f1..4270f45843122 100644 --- a/src/sources/docker_logs/tests.rs +++ b/src/sources/docker_logs/tests.rs @@ -30,12 +30,12 @@ mod integration_tests { } use bollard::{ + models::ContainerCreateBody, query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptionsBuilder, KillContainerOptions, ListImagesOptionsBuilder, RemoveContainerOptions, StartContainerOptions, WaitContainerOptions, }, - secret::ContainerCreateBody, }; use futures::{FutureExt, stream::TryStreamExt}; use itertools::Itertools as _; @@ -182,7 +182,7 @@ mod integration_tests { .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); - if let Some(error) = info.error { + if let Some(error) = info.error_detail { panic!("{error:?}"); } }) From 6267fcc78d8cd347f6405ab03ca100b97944a5ef Mon Sep 17 00:00:00 2001 From: bas smit <474727+fbs@users.noreply.github.com> Date: Thu, 26 Mar 2026 22:31:32 +0000 Subject: [PATCH 019/364] fix(opentelemetry source): log error (#24708) * fix(source opentelemetry): log error If the HTTP opentelemetry source fails to build it is currently silently ignored, without logging any info to the 'user'. The only indication you have that something is wrong is that the log line about the http listener starting is _not_ there. With this change the error condition is logged the same way the gRPC error is logged. Before: ``` $ vector -c vector-otel.yaml ... 2026-02-23T09:15:50.053251Z INFO vector: Vector has started. debug="false" version="0.52.0" arch="aarch64" revision="ca5bf26 2025-12-16 14:56:07.290167996" 2026-02-23T09:15:50.053468Z INFO source{component_kind="source" component_id=otel component_type=opentelemetry}: vector::sources::util::grpc: Building gRPC server. address=127.0.0.1:4317 ``` now: ``` $ target/debug/vector --config vector-otel.yaml 2026-02-23T09:16:49.397436Z INFO vector: Vector has started. debug="true" version="0.54.0" arch="aarch64" revision="" 2026-02-23T09:16:49.398497Z INFO source{component_kind="source" component_id=otel component_type=opentelemetry}: vector::sources::util::grpc: Building gRPC server. address=127.0.0.1:4317 2026-02-23T09:16:49.400243Z ERROR source{component_kind="source" component_id=otel component_type=opentelemetry}: vector::sources::opentelemetry::config: Source future failed. error=TCP bind failed: Address family not supported by protocol family (os error 47) ``` * improve error messages * add changelog * Fix CI errors Co-authored-by: Thomas * fix(opentelemetry source): improve error message wording Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Thomas Co-authored-by: Claude Sonnet 4.6 (1M context) --- changelog.d/24708_log_error.fix.md | 5 +++++ src/sources/opentelemetry/config.rs | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/24708_log_error.fix.md diff --git a/changelog.d/24708_log_error.fix.md b/changelog.d/24708_log_error.fix.md new file mode 100644 index 0000000000000..d53626b09af05 --- /dev/null +++ b/changelog.d/24708_log_error.fix.md @@ -0,0 +1,5 @@ +The `opentelemetry` source now logs an error if it fails to start up or during runtime. +This can happen when the configuration is invalid, for example trying to bind to the wrong +IP or when hitting the open file limit. + +authors: fbs diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 89f6163da0595..827bbd213360d 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -328,7 +328,7 @@ impl SourceConfig for OpentelemetryConfig { cx.shutdown.clone(), ) .map_err(|error| { - error!(message = "Source future failed.", %error); + error!(message = "OpenTelemetry source gRPC server failed.", %error); }); let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; @@ -355,7 +355,10 @@ impl SourceConfig for OpentelemetryConfig { filters, cx.shutdown, self.http.keepalive.clone(), - ); + ) + .map_err(|error| { + error!(message = "OpenTelemetry source HTTP server failed.", %error); + }); Ok(join(grpc_source, http_source).map(|_| Ok(())).boxed()) } From e8526b2a3ef7ac257d0fb816f611cddbc494a478 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 22:32:43 +0000 Subject: [PATCH 020/364] chore(deps): bump deadpool from 0.12.2 to 0.13.0 (#25014) * chore(deps): bump deadpool from 0.12.2 to 0.13.0 Bumps [deadpool](https://github.com/deadpool-rs/deadpool) from 0.12.2 to 0.13.0. - [Changelog](https://github.com/deadpool-rs/deadpool/blob/main/release.toml) - [Commits](https://github.com/deadpool-rs/deadpool/compare/deadpool-v0.12.2...deadpool-v0.13.0) --- updated-dependencies: - dependency-name: deadpool dependency-version: 0.13.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update Cargo.toml and licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 23 ++++++++++++++++++++--- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 4 ++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be47e7ba18697..0a3f2830ec51d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3196,7 +3196,18 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" dependencies = [ - "deadpool-runtime", + "deadpool-runtime 0.1.3", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime 0.3.1", "num_cpus", "tokio", ] @@ -3206,6 +3217,12 @@ name = "deadpool-runtime" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" dependencies = [ "tokio", ] @@ -12257,7 +12274,7 @@ dependencies = [ "criterion", "csv", "databend-client", - "deadpool", + "deadpool 0.13.0", "derivative", "dirs-next", "dnsmsg-parser", @@ -13844,7 +13861,7 @@ checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ "assert-json-diff", "base64 0.22.1", - "deadpool", + "deadpool 0.12.2", "futures", "http 1.3.1", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index a139a477d5ab2..7676ebe174ea3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,7 +336,7 @@ smpl_jwt = { version = "0.8.0", default-features = false, optional = true } # AMQP lapin = { version = "4.3.0", default-features = false, features = ["tokio", "native-tls"], optional = true } async-rs = { version = "0.8", default-features = false, features = ["tokio"], optional = true } -deadpool = { version = "0.12.2", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } +deadpool = { version = "0.13.0", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } # Opentelemetry diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8dbc10abc4f7e..9a135f0479f51 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -216,8 +216,8 @@ data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin databend-client,https://github.com/databendlabs/bendsql,Apache-2.0,Databend Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers -deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung -deadpool-runtime,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung +deadpool,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung +deadpool-runtime,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton From af15a5a595f342038fd3368854fa75c74c31e932 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 26 Mar 2026 21:09:12 -0400 Subject: [PATCH 021/364] fix(dev): vanila `cargo build` should run on windows (#24991) * fix(deps): gate Unix-only dependencies to enable Windows MSVC builds Move `tikv-jemallocator` to a Unix-only target dependency (it requires autoconf which is unavailable with MSVC). Enable `rdkafka/cmake_build` on Windows via a target-specific dependency entry so rdkafka builds using cmake instead of the Unix autotools path. Gate `sources-dnstap` module declarations on Unix to match the Unix-only `framestream` crate it depends on. Add `#[cfg(unix)]` to jemalloc allocator blocks in `src/lib.rs`. Tested on Windows with Cyrus SASL installed via MSYS2. Co-Authored-By: Claude Sonnet 4.6 * chore: apply rustfmt Co-Authored-By: Claude Sonnet 4.6 * refactor: make rdkafka a workspace dependency Avoids version string duplication across dependency sections. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: pront Co-authored-by: Claude Sonnet 4.6 --- Cargo.toml | 6 ++++-- src/internal_events/mod.rs | 4 ++-- src/lib.rs | 8 ++++++-- src/sources/mod.rs | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7676ebe174ea3..4bc52384f0a06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -182,6 +182,7 @@ prost-types = { version = "0.12", default-features = false } quickcheck = { version = "1.1.0" } rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } rand_distr = { version = "0.5.1", default-features = false } +rdkafka = { version = "0.39.0", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.37.0", default-features = false, features = ["std"] } @@ -411,7 +412,7 @@ pulsar = { version = "6.7.0", default-features = false, features = ["tokio-runti quick-junit = { version = "0.6.0" } rand.workspace = true rand_distr.workspace = true -rdkafka = { version = "0.39.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { workspace = true, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.32.4", default-features = false, features = ["connection-manager", "sentinel", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex.workspace = true roaring = { version = "0.11.2", default-features = false, features = ["std"], optional = true } @@ -425,7 +426,6 @@ sqlx = { version = "0.8.6", default-features = false, features = ["derive", "pos stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } -tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } tokio-postgres = { version = "0.7.13", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { workspace = true, features = ["connect"], optional = true } toml.workspace = true @@ -453,9 +453,11 @@ byteorder = "1.5.0" windows-service = "0.8.0" windows = { version = "0.58", features = ["Win32_System_EventLog", "Win32_Foundation", "Win32_System_Com", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_Storage_FileSystem"], optional = true } quick-xml = { version = "0.31", default-features = false, features = ["serialize"], optional = true } +rdkafka = { workspace = true, features = ["cmake_build"], optional = true } [target.'cfg(unix)'.dependencies] nix = { version = "0.31", default-features = false, features = ["socket", "signal", "fs"] } +tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } [target.'cfg(target_os = "linux")'.dependencies] procfs = { version = "0.18.0", default-features = false } diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 02b7460856f2d..d111134e97adf 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -40,7 +40,7 @@ mod datadog_traces; mod dedupe; #[cfg(feature = "sources-demo_logs")] mod demo_logs; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] mod dnstap; #[cfg(feature = "sources-docker_logs")] mod docker_logs; @@ -194,7 +194,7 @@ pub(crate) use self::datadog_traces::*; pub(crate) use self::dedupe::*; #[cfg(feature = "sources-demo_logs")] pub(crate) use self::demo_logs::*; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] pub(crate) use self::dnstap::*; #[cfg(feature = "sources-docker_logs")] pub(crate) use self::docker_logs::*; diff --git a/src/lib.rs b/src/lib.rs index d4d4ab1eeb38a..8d26a3b080ecf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,11 +35,15 @@ pub use indoc::indoc; // re-export codecs for convenience pub use vector_lib::codecs; -#[cfg(all(feature = "tikv-jemallocator", not(feature = "allocation-tracing")))] +#[cfg(all( + unix, + feature = "tikv-jemallocator", + not(feature = "allocation-tracing") +))] #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[cfg(all(feature = "tikv-jemallocator", feature = "allocation-tracing"))] +#[cfg(all(unix, feature = "tikv-jemallocator", feature = "allocation-tracing"))] #[global_allocator] static ALLOC: self::internal_telemetry::allocations::Allocator = self::internal_telemetry::allocations::get_grouped_tracing_allocator( diff --git a/src/sources/mod.rs b/src/sources/mod.rs index cdc03f46fdcbb..2384b096f630b 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -17,7 +17,7 @@ pub mod aws_sqs; pub mod datadog_agent; #[cfg(feature = "sources-demo_logs")] pub mod demo_logs; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] pub mod dnstap; #[cfg(feature = "sources-docker_logs")] pub mod docker_logs; From 4e6b49d6951b1f6ed9e2851aec674577ccc4c2b2 Mon Sep 17 00:00:00 2001 From: Yoenn Burban <62966490+gwenaskell@users.noreply.github.com> Date: Fri, 27 Mar 2026 12:47:55 +0100 Subject: [PATCH 022/364] fix(metrics): deduct downstream utilization on task transforms (#24731) * add output utilization on task transforms * add changelog * fix formatting * avoid pin_project on OutputUtilization * replace wrap functions with constructors * Apply suggestions from code review Co-authored-by: Pavlos Rontidis * fix formatting * woops * improve changelog --------- Co-authored-by: Pavlos Rontidis --- .../24641-task-transform-utilization.fix.md | 5 + src/topology/builder.rs | 11 +- src/utilization.rs | 223 ++++++++++++++++-- 3 files changed, 221 insertions(+), 18 deletions(-) create mode 100644 changelog.d/24641-task-transform-utilization.fix.md diff --git a/changelog.d/24641-task-transform-utilization.fix.md b/changelog.d/24641-task-transform-utilization.fix.md new file mode 100644 index 0000000000000..f59a9c8a8eddf --- /dev/null +++ b/changelog.d/24641-task-transform-utilization.fix.md @@ -0,0 +1,5 @@ +Fixed utilization for task transforms to not account for time spent when downstream +is not polling. If the transform is frequently blocked on downstream components, +the reported utilization should be lower. + +authors: gwenaskell diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 4fda56f9b35b3..8048b7a6073ed 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -52,7 +52,10 @@ use crate::{ spawn_named, topology::task::TaskError, transforms::{SyncTransform, TaskTransform, Transform, TransformOutputs, TransformOutputsBuf}, - utilization::{UtilizationComponentSender, UtilizationEmitter, UtilizationRegistry, wrap}, + utilization::{ + OutputUtilization, Utilization, UtilizationComponentSender, UtilizationEmitter, + UtilizationRegistry, + }, }; static ENRICHMENT_TABLES: LazyLock = @@ -641,7 +644,7 @@ impl<'a> Builder<'a> { .take() .expect("Task started but input has been taken."); - let mut rx = wrap(utilization_sender, component_key.clone(), rx); + let mut rx = Utilization::new(utilization_sender, component_key.clone(), rx); let events_received = register!(EventsReceived); sink.run( @@ -801,7 +804,8 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry .add_component(key.clone(), gauge!("utilization")); - let input_rx = wrap(sender, key.clone(), input_rx.into_stream()); + let output_sender = sender.clone(); + let input_rx = Utilization::new(sender, key.clone(), input_rx.into_stream()); let events_received = register!(EventsReceived); let filtered = input_rx @@ -846,6 +850,7 @@ impl<'a> Builder<'a> { events.estimated_json_encoded_size_of(), )); }); + let stream = OutputUtilization::new(output_sender, stream); let transform = async move { debug!("Task transform starting."); diff --git a/src/utilization.rs b/src/utilization.rs index 413505b8fc9f5..b3496b998f1c5 100644 --- a/src/utilization.rs +++ b/src/utilization.rs @@ -24,6 +24,11 @@ use vector_lib::{id::ComponentKey, shutdown::ShutdownSignal, stats}; const UTILIZATION_EMITTER_DURATION: Duration = Duration::from_secs(5); +/// Stream wrappers used to approximate component utilization from poll timing. +/// +/// Current model: +/// +/// #[pin_project] pub(crate) struct Utilization { intervals: IntervalStream, @@ -33,6 +38,23 @@ pub(crate) struct Utilization { } impl Utilization { + /// Output-side utilization wrapper for task transforms. + /// + /// Measures the time after the wrapped stream yields an item until downstream + /// polls it again. + pub(crate) fn new( + timer_tx: UtilizationComponentSender, + component_key: ComponentKey, + inner: S, + ) -> Self { + Self { + intervals: IntervalStream::new(interval(Duration::from_secs(5))), + timer_tx, + component_key, + inner, + } + } + /// Consumes this wrapper and returns the inner stream. /// /// This can't be constant because destructors can't be run in a const context, and we're @@ -160,6 +182,7 @@ enum UtilizationTimerMessage { StopWait(ComponentKey, Instant), } +#[derive(Clone)] pub(crate) struct UtilizationComponentSender { component_key: ComponentKey, timer_tx: Sender, @@ -277,23 +300,43 @@ impl UtilizationEmitter { } } -/// Wrap a stream to emit stats about utilization. This is designed for use with -/// the input channels of transform and sinks components, and measures the -/// amount of time that the stream is waiting for input from upstream. We make -/// the simplifying assumption that this wait time is when the component is idle -/// and the rest of the time it is doing useful work. This is more true for -/// sinks than transforms, which can be blocked by downstream components, but -/// with knowledge of the config the data is still useful. -pub(crate) fn wrap( +/// Output-side counterpart of [`Utilization`]. Wraps the output stream of a +/// task transform to track time spent waiting for downstream to accept items. +/// +/// While [`Utilization`] measures idle time as "waiting for upstream input", +/// this wrapper measures the complementary case: after yielding an item, the +/// time until the consumer polls again is counted as downstream wait (idle). +/// Both wrappers share a single [`Timer`] per component, and the idempotent +/// `start_wait`/`stop_wait` transitions ensure no double-counting. +pub(crate) struct OutputUtilization { timer_tx: UtilizationComponentSender, - component_key: ComponentKey, inner: S, -) -> Utilization { - Utilization { - intervals: IntervalStream::new(interval(Duration::from_secs(5))), - timer_tx, - component_key, - inner, +} + +impl Stream for OutputUtilization +where + S: Stream + Unpin, +{ + type Item = S::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + this.timer_tx.try_send_stop_wait(); + let result = ready!(this.inner.poll_next_unpin(cx)); + if result.is_some() { + this.timer_tx.try_send_start_wait(); + } + Poll::Ready(result) + } +} + +impl OutputUtilization { + /// Wrap a task transform stream to track time spent waiting for downstream + /// to consume items. This is the output-side counterpart to + /// [`Utilization::new`], designed for use with task transforms where the + /// framework cannot otherwise detect downstream backpressure. + pub(crate) const fn new(timer_tx: UtilizationComponentSender, inner: S) -> Self { + Self { timer_tx, inner } } } @@ -429,4 +472,154 @@ mod tests { let avg = timer.ewma.average().unwrap(); assert_approx_eq(avg, 1.0, "near 1.0 (never waiting)"); } + + /// Mock task transform that passes events through unchanged, simulating + /// a configurable processing delay by advancing MockClock per item. + struct MockTaskTransform { + processing_time: Duration, + } + + use crate::event::EventArray; + use crate::transforms::TaskTransform; + + impl TaskTransform for MockTaskTransform { + fn transform( + self: Box, + task: Pin + Send>>, + ) -> Pin + Send>> { + let processing_time = self.processing_time; + task.map(move |events| { + MockClock::advance(processing_time); + events + }) + .boxed() + } + } + + /// End-to-end test exercising the Utilization (input) and + /// OutputUtilization (output) stream wrappers with a mock TaskTransform, + /// wired up the same way `build_task_transform` does in the builder. + /// + /// Pipeline: channel → Utilization(input) → TaskTransform → OutputUtilization(output) + /// + /// Timeline (10s): + /// T=100..103 waiting for input (3s wait) + /// T=103..105 transform processing (2s work) + /// T=105..108 blocked on downstream (3s wait, even though input has data) + /// T=108..110 transform processing (2s work) + /// + /// Expected utilization = 4/10 = 0.4 + #[tokio::test] + #[serial] + async fn test_task_transform_utilization_end_to_end() { + use crate::event::{EventArray, LogEvent}; + use crate::transforms::TaskTransform; + use futures::SinkExt; + use futures::channel::mpsc as futures_mpsc; + + MockClock::set_time(Duration::from_secs(100)); + + let (mut emitter, registry) = UtilizationEmitter::new(); + let key = ComponentKey::from("test_transform"); + + let sender = registry.add_component(key.clone(), metrics::gauge!("utilization")); + let output_sender = sender.clone(); + + // Upstream channel carrying EventArrays. + let (mut input_tx, input_rx) = futures_mpsc::channel::(10); + + // Wire up the pipeline exactly like build_task_transform: + // Utilization(input) → transform.transform() → OutputUtilization(output) + let input_wrapped = Utilization::new(sender, key.clone(), input_rx); + let transform = Box::new(MockTaskTransform { + processing_time: Duration::from_secs(2), + }); + let transform_output = transform.transform(Box::pin(input_wrapped)); + let mut pipeline = OutputUtilization::new(output_sender, transform_output); + + let waker = futures::task::noop_waker(); + let mut cx = std::task::Context::from_waker(&waker); + + // -- Phase 1: poll transform, no input available -- + // T=100: start_wait sent by input wrapper. + assert!(Pin::new(&mut pipeline).poll_next(&mut cx).is_pending()); + MockClock::advance(Duration::from_secs(3)); + // T=103: 3s of input wait. Period: 0s work / 3s → util = 0.0. + check_timer_utilization(&mut emitter, &key, 0.0, "0.0 (phase 1: all input wait)"); + + // -- Phase 2: send input, poll pipeline → transform processes -- + input_tx + .send(EventArray::from(LogEvent::default())) + .await + .unwrap(); + assert!(Pin::new(&mut pipeline).poll_next(&mut cx).is_ready()); + // T=105: MockClock advanced 2s inside transform. start_wait sent by + // output wrapper. Period: 2s work / 0s wait → util = 1.0. + // EWMA: 0.9×1.0 + 0.1×0.0 = 0.9 + check_timer_utilization(&mut emitter, &key, 0.9, "0.9 (phase 2: all work)"); + + // -- Phase 3: send another event, simulate downstream blocking -- + // Data is available in the input channel, but the transform is not + // polled so no progress is made: the time counts as downstream wait. + input_tx + .send(EventArray::from(LogEvent::default())) + .await + .unwrap(); + MockClock::advance(Duration::from_secs(3)); + // T=108: 3s downstream wait. Period: 0s work / 3s → util = 0.0. + // EWMA: 0.9×0.0 + 0.1×0.9 = 0.09 + check_timer_utilization( + &mut emitter, + &key, + 0.09, + "0.09 (phase 3: downstream wait despite buffered input)", + ); + + // -- Phase 4: poll pipeline, processes the queued event -- + assert!(Pin::new(&mut pipeline).poll_next(&mut cx).is_ready()); + // T=110: MockClock advanced 2s inside transform. Period: 2s work / 0s wait → util = 1.0. + // EWMA: 0.9×1.0 + 0.1×0.09 ≈ 0.909 + check_timer_utilization( + &mut emitter, + &key, + 0.909, + "0.909 (phase 4: queued event processed)", + ); + } + + /// Drain pending timer messages and assert the current EWMA utilization + /// for a single reporting period. + fn check_timer_utilization( + emitter: &mut UtilizationEmitter, + key: &ComponentKey, + expected: f64, + description: &str, + ) { + drain_emitter_messages(emitter); + let mut timers = emitter.timers.lock().expect("mutex poisoned"); + let timer = timers.get_mut(key).expect("timer should exist"); + timer.update_utilization(); + let avg = timer.ewma.average().unwrap(); + assert_approx_eq(avg, expected, description); + } + + /// Drain all pending messages from the emitter's channel into the timers, + /// simulating what `run_utilization` does in a loop. + fn drain_emitter_messages(emitter: &mut UtilizationEmitter) { + while let Ok(message) = emitter.timer_rx.try_recv() { + let mut timers = emitter.timers.lock().expect("mutex poisoned"); + match message { + UtilizationTimerMessage::StartWait(key, at) => { + if let Some(timer) = timers.get_mut(&key) { + timer.start_wait(at); + } + } + UtilizationTimerMessage::StopWait(key, at) => { + if let Some(timer) = timers.get_mut(&key) { + timer.stop_wait(at); + } + } + } + } + } } From a7450cdbba3eb86782d67bea39b81e5b641067d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:50:50 -0400 Subject: [PATCH 023/364] chore(deps): bump tempfile from 3.23.0 to 3.27.0 (#25016) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.23.0 to 3.27.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.23.0...v3.27.0) --- updated-dependencies: - dependency-name: tempfile dependency-version: 3.27.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a3f2830ec51d..5bbba19140aa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,7 +765,7 @@ dependencies = [ "cfg-if", "event-listener 5.3.1", "futures-lite", - "rustix 1.0.1", + "rustix 1.1.4", ] [[package]] @@ -2925,7 +2925,7 @@ dependencies = [ "futures-core", "mio", "parking_lot", - "rustix 1.0.1", + "rustix 1.1.4", "signal-hook", "signal-hook-mio", "winapi", @@ -6213,9 +6213,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "linux-raw-sys" -version = "0.9.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "listenfd" @@ -8088,7 +8088,7 @@ checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" dependencies = [ "bitflags 2.10.0", "procfs-core", - "rustix 1.0.1", + "rustix 1.1.4", ] [[package]] @@ -9397,15 +9397,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags 2.10.0", "errno", "libc", - "linux-raw-sys 0.9.2", - "windows-sys 0.59.0", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.0", ] [[package]] @@ -10890,14 +10890,14 @@ checksum = "83176759e9416cf81ee66cb6508dbfe9c96f20b8b56265a39917551c23c70964" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", - "rustix 1.0.1", + "rustix 1.1.4", "windows-sys 0.61.0", ] diff --git a/Cargo.toml b/Cargo.toml index 4bc52384f0a06..b2670983edbe2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -192,7 +192,7 @@ serde_json = { version = "1.0.143", default-features = false, features = ["prese serde_yaml = { version = "0.9.34", default-features = false } snafu = { version = "0.9.0", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } -tempfile = "3.23.0" +tempfile = "3.27.0" tokio = { version = "1.49.0", default-features = false } tokio-stream = { version = "0.1.18", default-features = false } tokio-test = "0.4.5" From 7066adccecc24fe005f1fb7c934bbda4b557ea6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:51:39 -0400 Subject: [PATCH 024/364] chore(deps): bump proptest from 1.10.0 to 1.11.0 (#25021) * chore(deps): bump proptest from 1.10.0 to 1.11.0 Bumps [proptest](https://github.com/proptest-rs/proptest) from 1.10.0 to 1.11.0. - [Release notes](https://github.com/proptest-rs/proptest/releases) - [Changelog](https://github.com/proptest-rs/proptest/blob/main/CHANGELOG.md) - [Commits](https://github.com/proptest-rs/proptest/compare/v1.10.0...v1.11.0) --- updated-dependencies: - dependency-name: proptest dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Use workspace proptest --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 4 ++-- lib/vector-stream/Cargo.toml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5bbba19140aa9..3c1cd0979e24d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8116,9 +8116,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", diff --git a/Cargo.toml b/Cargo.toml index b2670983edbe2..d4c3fb4c46c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -173,7 +173,7 @@ nom = { version = "8.0.0", default-features = false } ordered-float = { version = "4.6.0", default-features = false } pastey = { version = "0.2", default-features = false } pin-project = { version = "1.1.11", default-features = false } -proptest = { version = "1.10" } +proptest = { version = "1.11" } proptest-derive = { version = "0.6.0" } prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 04ad96436baf1..4f27f3ef369e1 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -42,7 +42,7 @@ crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } -proptest = "1.10" +proptest.workspace = true quickcheck.workspace = true rand.workspace = true serde_yaml.workspace = true diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index a7e33377bc09c..41ad3d4b86b5b 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -37,7 +37,7 @@ ordered-float.workspace = true openssl = { version = "0.10.75", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.5", default-features = false } pin-project.workspace = true -proptest = { version = "1.10", optional = true } +proptest = { workspace = true, optional = true } prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } @@ -82,7 +82,7 @@ criterion = { workspace = true, features = ["html_reports"] } env-test-util = "1.0.1" quickcheck.workspace = true quickcheck_macros = "1" -proptest = "1.10" +proptest.workspace = true similar-asserts = "1.7.0" tokio-test.workspace = true toml.workspace = true diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index b16138436eb82..21b9f1a9a2d7b 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -19,7 +19,7 @@ vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.10" +proptest.workspace = true rand.workspace = true rand_distr.workspace = true tokio = { workspace = true, features = ["test-util"] } From 3ca3374c5b0359a3839f6c8091e2c3632faec719 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 15:02:57 +0000 Subject: [PATCH 025/364] chore(deps): bump rust_decimal from 1.39.0 to 1.40.0 (#24799) Bumps [rust_decimal](https://github.com/paupino/rust-decimal) from 1.39.0 to 1.40.0. - [Release notes](https://github.com/paupino/rust-decimal/releases) - [Changelog](https://github.com/paupino/rust-decimal/blob/master/CHANGELOG.md) - [Commits](https://github.com/paupino/rust-decimal/compare/1.39.0...1.40.0) --- updated-dependencies: - dependency-name: rust_decimal dependency-version: 1.40.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c1cd0979e24d..a9809924cfd68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9343,9 +9343,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.39.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" +checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" dependencies = [ "arrayvec", "borsh", diff --git a/Cargo.toml b/Cargo.toml index d4c3fb4c46c65..21c40c793a9bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -185,7 +185,7 @@ rand_distr = { version = "0.5.1", default-features = false } rdkafka = { version = "0.39.0", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } -rust_decimal = { version = "1.37.0", default-features = false, features = ["std"] } +rust_decimal = { version = "1.40.0", default-features = false, features = ["std"] } semver = { version = "1.0.27", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } From c1f34ef803fe5365a5be54f46877590bc8c91ba6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:44:05 +0000 Subject: [PATCH 026/364] chore(website deps): bump brace-expansion from 1.1.12 to 1.1.13 in /website (#25056) chore(website deps): bump brace-expansion in /website Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 1.1.12 to 1.1.13. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/v1.1.12...v1.1.13) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 1.1.13 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index f7cedc6f6748f..da88eb1b5980f 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -1744,9 +1744,9 @@ boolbase@^1.0.0: integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24= brace-expansion@^1.1.7: - version "1.1.12" - resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz" - integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg== + version "1.1.13" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.13.tgz#d37875c01dc9eff988dd49d112a57cb67b54efe6" + integrity sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" From d29a524ea05dcbb6f823121ffdff449ecd960673 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:49:17 +0000 Subject: [PATCH 027/364] chore(deps): bump security-framework from 3.5.1 to 3.6.0 (#24763) Bumps [security-framework](https://github.com/kornelski/rust-security-framework) from 3.5.1 to 3.6.0. - [Commits](https://github.com/kornelski/rust-security-framework/compare/v3.5.1...v3.6.0) --- updated-dependencies: - dependency-name: security-framework dependency-version: 3.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ lib/vector-core/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9809924cfd68..a557cd2ced9e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9471,7 +9471,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework 3.6.0", ] [[package]] @@ -9711,9 +9711,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ "bitflags 2.10.0", "core-foundation 0.10.1", @@ -9724,9 +9724,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -12626,7 +12626,7 @@ dependencies = [ "regex", "ryu", "schannel", - "security-framework 3.5.1", + "security-framework 3.6.0", "serde", "serde_json", "serde_with", diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 41ad3d4b86b5b..8a512884b2c32 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -67,7 +67,7 @@ vrl.workspace = true cfg-if.workspace = true [target.'cfg(target_os = "macos")'.dependencies] -security-framework = "3.5.1" +security-framework = "3.6.0" [target.'cfg(windows)'.dependencies] schannel = "0.1.28" From 89e5c343c631e36090d2543106b88685d7a1c908 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:55:57 -0400 Subject: [PATCH 028/364] chore(ci): bump juliangruber/read-file-action from 1.1.7 to 1.1.8 (#25000) Bumps [juliangruber/read-file-action](https://github.com/juliangruber/read-file-action) from 1.1.7 to 1.1.8. - [Release notes](https://github.com/juliangruber/read-file-action/releases) - [Commits](https://github.com/juliangruber/read-file-action/compare/b549046febe0fe86f8cb4f93c24e284433f9ab58...271ff311a4947af354c6abcd696a306553b9ec18) --- updated-dependencies: - dependency-name: juliangruber/read-file-action dependency-version: 1.1.8 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index f2216df39e684..1b93487a54ec2 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -538,7 +538,7 @@ jobs: - name: Read regression report id: read-analysis - uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7 + uses: juliangruber/read-file-action@271ff311a4947af354c6abcd696a306553b9ec18 # v1.1.8 with: path: ${{ runner.temp }}/outputs/report.md From 55be10e5860afd47656a67a08335bb17a1d641c8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 12:14:44 -0400 Subject: [PATCH 029/364] fix(ci): add missing `contents: read` permission to integration review jobs (#25067) fix(ci): add missing contents:read permission to integration review jobs --- .github/workflows/ci-integration-review.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 1195b4372d97b..a2a73c39e7d0e 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -85,6 +85,7 @@ jobs: build-test-runner: needs: prep-pr permissions: + contents: read packages: write # Required to push test runner image to GHCR uses: ./.github/workflows/build-test-runner.yml with: @@ -98,6 +99,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 permissions: + contents: read packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false @@ -140,6 +142,7 @@ jobs: runs-on: ubuntu-24.04-8core timeout-minutes: 30 permissions: + contents: read packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false From d906ba19487c0552be7557d49a3f1dbfcd2405d9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:12:33 -0400 Subject: [PATCH 030/364] chore(deps): upgrade rustls from 0.23.23 to 0.23.37 (#25075) Upgrade rustls from 0.23.23 to 0.23.37 --- Cargo.lock | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a557cd2ced9e5..76e4c912d09ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1903,7 +1903,7 @@ dependencies = [ "hyperlocal", "log", "pin-project-lite", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -5147,7 +5147,7 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "hyper-util", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -6735,7 +6735,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.23.23", + "rustls 0.23.37", "rustversion", "serde", "serde_bytes", @@ -8505,7 +8505,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.23", + "rustls 0.23.37", "socket2 0.5.10", "thiserror 2.0.17", "tokio", @@ -8524,7 +8524,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -9094,7 +9094,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -9436,15 +9436,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.10", "subtle", "zeroize", ] @@ -9495,11 +9495,12 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", + "zeroize", ] [[package]] @@ -9523,6 +9524,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -10378,7 +10390,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.23", + "rustls 0.23.37", "serde", "serde_json", "sha2", @@ -11230,7 +11242,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.23", + "rustls 0.23.37", "tokio", ] From 6a50bd50c9d482511f5a2a21466a3427dd752263 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:27:02 -0400 Subject: [PATCH 031/364] chore(internal docs): update security policy structure and content (#25074) * docs(security): update security policy structure and content * Re-add meta/review schedule * Add whitespace * Revert changes to User Privileges * Finish CI/CD paragraph --- SECURITY.md | 102 ++++++++++++++++++++++------------------------------ 1 file changed, 43 insertions(+), 59 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 667a3f7b19d8e..e56f752785d91 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,9 +19,11 @@ possible on our security efforts. - [Open Source](#open-source) - [Workflow](#workflow) - [Version Control](#version-control) - - [Git](#git) + - [Pull Requests](#pull-requests) + - [Reviews & Approvals](#reviews--approvals) - [Signed Commits](#signed-commits) - [Protected Branches](#protected-branches) + - [Merge Policies](#merge-policies) - [Personnel](#personnel) - [Education](#education) - [Policies](#policies) @@ -34,26 +36,22 @@ possible on our security efforts. - [Unsafe Code](#unsafe-code) - [User Privileges](#user-privileges) - [Dependencies](#dependencies) - - [Change Control](#change-control) - - [Pull Requests](#pull-requests) - - [Reviews & Approvals](#reviews--approvals) - - [Merge Policies](#merge-policies) - [Automated Checks](#automated-checks) - [Vulnerability Scans & Security Advisories](#vulnerability-scans--security-advisories) - [Vulnerability Remediation](#vulnerability-remediation) - - [Fuzz Testing](#fuzz-testing) - [Infrastructure](#infrastructure) - [CI/CD](#cicd) - - [Runtime Isolation](#runtime-isolation) - [Network Security](#network-security) - - [Penetration Testing](#penetration-testing) - [Protocols](#protocols) - [Release Artifacts & Channels](#release-artifacts--channels) - [Asset Audit Logging](#asset-audit-logging) - [Asset Signatures & Checksums](#asset-signatures--checksums) +- [Vulnerability Reporting](#vulnerability-reporting) - [Meta](#meta) - [Review Schedule](#review-schedule) - [Vulnerability Reporting](#vulnerability-reporting) + - [Vector CI](#vector-ci) + - [Other reports](#other-reports) ## Project Structure @@ -84,10 +82,19 @@ are all publicly available. Version control ensures that all code changes are audited and authentic. -#### Git +Vector uses [Git][urls.git] to ensure that changes are auditable and traceable. + +#### Pull Requests + +All changes to Vector must go through a pull request review process. + +#### Reviews & Approvals + +All pull requests must be reviewed by at least one Vector team member. The +review process takes into account many factors, all of which are detailed in +our [Reviewing guide](REVIEWING.md). In exceptional circumstances, this +approval can be retroactive. -Vector leverages the [Git][urls.git] version-control system. This ensures all -changes are audited and traceable. #### Signed Commits @@ -107,6 +114,12 @@ are [protected][urls.github_protected_branches]. The exact requirements are: - Signed commits are required. - Administrators are included in these checks. +#### Merge Policies + +Vector requires pull requests to pass all [automated checks](#automated-checks). +Once passed, the pull request must be squashed and merged. This creates a clean +linear history with a Vector team member's co-sign. + ## Personnel ### Education @@ -116,7 +129,7 @@ the [contributing](CONTRIBUTING.md) and [reviewing](REVIEWING.md) documents. ### Policies -Vector maintains this security policy. Changed are communicated to all Vector +Vector maintains this security policy. Changes are communicated to all Vector team members. ### Two-factor Authentication @@ -150,8 +163,9 @@ catch many common sources of vulnerabilities at compile time. #### Unsafe Code -Vector does not allow the use of unsafe code except in circumstances where it -is required, such as dealing with CFFI. +Vector uses unsafe code sparingly. Unsafe is sometimes required, such as dealing +with CFFI. We may occasionally also use unsafe code for performance reasons but +those changes are kept to a minimum. #### User Privileges @@ -164,27 +178,6 @@ Vector aims to reduce the number of dependencies it relies on. If a dependency is added it goes through a comprehensive review process that is detailed in the [Reviewing guide](REVIEWING.md#dependencies). -### Change Control - -As noted above Vector uses the Git version control system on GitHub. - -#### Pull Requests - -All changes to Vector must go through a pull request review process. - -#### Reviews & Approvals - -All pull requests must be reviewed by at least one Vector team member. The -review process takes into account many factors, all of which are detailed in -our [Reviewing guide](REVIEWING.md). In exceptional circumstances, this -approval can be retroactive. - -#### Merge Policies - -Vector requires pull requests to pass all [automated checks](#automated-checks). -Once passed, the pull request must be squashed and merged. This creates a clean -linear history with a Vector team member's co-sign. - ### Automated Checks When possible, we'll create automated checks to enforce security policies. @@ -195,25 +188,21 @@ When possible, we'll create automated checks to enforce security policies. is part of the [Rust Security advisory database][urls.rust_sec]. The configuration, and a list of currently accepted advisories, are maintained in the [Cargo Deny configuration][urls.cargo_deny_configuration]. The check is run - [on every incoming PR][urls.cargo_deny_schedule] to the Vector project. + on every PR to the Vector project. - Vector implements [Dependabot][urls.dependabot] which performs automated upgrades on dependencies and [alerts][urls.dependabot_alerts] about any dependency-related security vulnerabilities. #### Vulnerability Remediation -If the advisory check fails then the PR will not be merged. We review each advisory to -determine what action to take. If possible, we update the dependency to a version -where the vulnerability has been addressed. If this isn't possible we either record -the acceptance of the vulnerability or replace the dependency. If we accept the -vulnerability we open a ticket to track its remediation, generally awaiting a fix -upstream. If the risk is deemed unacceptable we revisit the code and dependency -to find a more secure alternative. - -#### Fuzz Testing - -Vector implements automated fuzz testing to probe our code for other sources -of potential vulnerabilities. +If the advisory check fails due to changes made in the PR, it will not be +merged. We review each advisory to determine what action to take. Whenever +possible, we update the dependency to a version where the vulnerability has been +addressed. If this isn't possible we either record the acceptance of the +vulnerability or replace the dependency. If we accept the vulnerability we open +a ticket to track its remediation, generally awaiting a fix upstream. If the +risk is deemed unacceptable we revisit the code and dependency to find a more +secure alternative. ## Infrastructure @@ -223,16 +212,12 @@ Vector's infrastructure and how we secure them. ### CI/CD -#### Runtime Isolation - -All builds run in an isolated sandbox that is destroyed after each use. +All builds run in GitHub Actions runners which are ephemeral and don't maintain +state after the job is completed. We ensure we are following [OpenSSF best +practices](https://bestpractices.dev/) to minimize CI risk and exposure. ### Network Security -#### Penetration Testing - -Vector performs quarterly pen tests on vector.dev. - #### Protocols All network traffic is secured via TLS and SSH. This includes checking out @@ -260,7 +245,7 @@ Vector reviews this policy and all user access levels on a quarterly basis. We deeply appreciate any effort to discover and disclose security vulnerabilities responsibly. -## Vector CI +#### Vector CI If you would like to report a Vector CI vulnerability or have any security concerns with other Datadog products, please e-mail security@datadoghq.com. @@ -270,9 +255,9 @@ and verify the vulnerability before taking the necessary steps to fix it. After our initial reply to your disclosure, which should be directly after receiving it, we will periodically update you with the status of the fix. -## Other reports +#### Other reports -Due to the nature of a open-source project, Vector deployments are fully managed by users. Thus vulnerabilities in Vector deployments could +Due to the nature of an open-source project, Vector deployments are fully managed by users. Thus vulnerabilities in Vector deployments could potentially be exploited by malicious actors who already have access to the user’s infrastructure. We encourage responsible disclosure via opening an [open an issue][urls.new_security_report] so that risks can be properly assessed and mitigated. @@ -286,7 +271,6 @@ following when reporting: [urls.cargo_deny]: https://github.com/EmbarkStudios/cargo-deny [urls.cargo_deny_configuration]: https://github.com/vectordotdev/vector/blob/master/deny.toml -[urls.cargo_deny_schedule]: https://github.com/vectordotdev/vector/blob/master/.github/workflows/test.yml#L267 [urls.dependabot]: https://github.com/marketplace/dependabot-preview [urls.dependabot_alerts]: https://github.com/vectordotdev/vector/network/alerts [urls.git]: https://git-scm.com/ From 122cca4678172bbc0c067c65fcc909c6b7aa9ce6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 30 Mar 2026 17:51:55 -0400 Subject: [PATCH 032/364] chore(ci): update GHCR cleanup workflow (#25037) * chore(ci): improve GHCR cleanup workflow - Add step to delete dated nightly tags older than 30 days, guarded against semver/release tags - Increase test-runner cleanup batch size from 50 to 100 - Add continue-on-error to test-runner step - Update workflow name and header comment Co-Authored-By: Claude Sonnet 4.6 * chore(ci): replace gh CLI step with action for dated nightly cleanup Co-Authored-By: Claude Sonnet 4.6 * use ubuntu-24.04 * keep 30 nightlies --------- Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/cleanup-ghcr-images.yml | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cleanup-ghcr-images.yml b/.github/workflows/cleanup-ghcr-images.yml index c11ecbf6a0c9d..dced4f4f0d449 100644 --- a/.github/workflows/cleanup-ghcr-images.yml +++ b/.github/workflows/cleanup-ghcr-images.yml @@ -2,39 +2,43 @@ # # This workflow cleans up old images from GitHub Container Registry # to prevent unlimited storage growth. It runs weekly and removes: -# 1. Untagged images from all packages (intermediate build artifacts) -# 2. Old test-runner versions (keeps 50 most recent, deletes up to 50 oldest) +# 1. Old dated nightly tags for vector (keeps last ~8) +# 2. Old test-runner versions (keeps 5 most recent) -name: Cleanup Untagged GHCR Images +name: Cleanup GHCR Images on: schedule: # Run weekly on Sundays at 2 AM UTC - - cron: '0 2 * * 0' + - cron: "0 2 * * 0" workflow_dispatch: permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: - cleanup: - runs-on: ubuntu-latest + cleanup-vector-nightlies: + runs-on: ubuntu-24.04 permissions: - packages: write # Required to delete package versions from GHCR + packages: write # Required to delete package versions from GHCR steps: - - name: Delete untagged vector images + - name: Delete old dated nightly vector images uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0 with: package-name: vector package-type: container - min-versions-to-keep: 0 - delete-only-untagged-versions: true - continue-on-error: true + min-versions-to-keep: 30 + ignore-versions: '^(nightly$|\d+\.\d+)' - - name: Delete old vector-test-runner images + cleanup-test-runner: + runs-on: ubuntu-24.04 + permissions: + packages: write # Required to delete package versions from GHCR + steps: + - name: Delete old test-runner images uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0 with: package-name: test-runner package-type: container min-versions-to-keep: 5 - num-old-versions-to-delete: 50 + num-old-versions-to-delete: 100 From 87ed519cc40011224208914f1dd79b7240f7e1d7 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 31 Mar 2026 09:36:52 -0400 Subject: [PATCH 033/364] chore(dev): consolidate advisory ignores with issue links (#25076) chore(deny): consolidate advisory ignores with issue links --- deny.toml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/deny.toml b/deny.toml index 5a3f7dee13c4d..b8e0be77f41c8 100644 --- a/deny.toml +++ b/deny.toml @@ -40,16 +40,8 @@ license-files = [ [advisories] ignore = [ - # Vulnerability in `rsa` crate: https://rustsec.org/advisories/RUSTSEC-2023-0071.html - # There is not fix available yet. - # https://github.com/vectordotdev/vector/issues/19262 - "RUSTSEC-2023-0071", - { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained" }, - { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained" }, - { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, - # rustls-pemfile is unmaintained. Blocked by both async-nats and http 1.0.0 upgrade. - { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained" }, - # rustls-webpki 0.101.7 vulnerability. Fix requires upgrading rustls from 0.21 to 0.23+, - # which is a significant chain upgrade through aws-smithy-http-client, hyper-rustls, tokio-rustls, etc. - { id = "RUSTSEC-2026-0049", reason = "Fix requires major rustls upgrade (0.21 -> 0.23+); tracked for future upgrade" }, + { id = "RUSTSEC-2023-0071", reason = "rsa marvin attack - unpatched upstream (https://github.com/vectordotdev/vector/issues/19262)" }, + { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, + { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, ] From 84c7af2264deacb1afc0529671d53c8c6792bf88 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 31 Mar 2026 09:41:49 -0400 Subject: [PATCH 034/364] fix(ci): install vdev via setup action in ci-integration-review workflow (#25077) chore(ci): install vdev via setup action in ci-integration-review workflow --- .github/workflows/ci-integration-review.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index a2a73c39e7d0e..30e359e366f30 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -117,14 +117,17 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/setup + with: + vdev: true + datadog-ci: true + - name: Pull test runner image uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - - run: bash scripts/environment/prepare.sh --modules=datadog-ci - - name: Integration Tests - ${{ matrix.service }} if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') @@ -156,14 +159,17 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/setup + with: + vdev: true + datadog-ci: true + - name: Pull test runner image uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - - run: bash scripts/environment/prepare.sh --modules=datadog-ci - - name: E2E Tests - ${{ matrix.service }} if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') From d732259aaa6b5e156b001672ccf0e59be5e84802 Mon Sep 17 00:00:00 2001 From: StepSecurity Bot Date: Tue, 31 Mar 2026 10:08:30 -0700 Subject: [PATCH 035/364] chore(ci): pin GitHub actions to full sha (#25080) Signed-off-by: StepSecurity Bot --- .github/workflows/integration_windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_windows.yml b/.github/workflows/integration_windows.yml index 5e89827f52e3a..39289d786fd24 100644 --- a/.github/workflows/integration_windows.yml +++ b/.github/workflows/integration_windows.yml @@ -16,7 +16,7 @@ jobs: windows: ${{ steps.filter.outputs.windows }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: filters: | From 0943dd938f247b0a1963e6bcb1a464ae49392c5f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 31 Mar 2026 14:54:02 -0400 Subject: [PATCH 036/364] chore(ci): migrate from markdownlint-cli to markdownlint-cli2 (#25081) * chore(ci): migrate from markdownlint-cli to markdownlint-cli2 markdownlint-cli is the legacy CLI wrapper. markdownlint-cli2 is the modern replacement used by all major editors (Zed, VS Code, JetBrains) and auto-detects .markdownlint.jsonc without an explicit --config flag. This unifies the linting tool between CI and local editor environments, so developers see the same results in their IDE as in CI. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(ci): rename markdownlint input to markdownlint-cli2 Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/actions/setup/action.yml | 8 ++++---- .github/workflows/test.yml | 2 +- .markdownlint.jsonc | 5 +---- Makefile | 2 +- scripts/environment/prepare.sh | 8 ++++---- vdev/src/commands/check/markdown.rs | 7 ++----- 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 379afd232341c..f9037aebb68e8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -76,10 +76,10 @@ inputs: description: "Install vdev CLI tool (cached by vdev/ directory changes)." # prepare.sh - npm - markdownlint: + markdownlint-cli2: required: false default: false - description: "Install markdownlint (npm)." + description: "Install markdownlint-cli2 (npm)." datadog-ci: required: false default: false @@ -267,7 +267,7 @@ runs: ~/.cargo/bin/cargo-llvm-cov ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack - /usr/local/bin/markdownlint + /usr/local/bin/markdownlint-cli2 /usr/local/bin/datadog-ci key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} restore-keys: | @@ -287,7 +287,7 @@ runs: [[ "${{ inputs.cargo-llvm-cov }}" == "true" ]] && mods+=("cargo-llvm-cov") [[ "${{ inputs.dd-rust-license-tool }}" == "true" ]] && mods+=("dd-rust-license-tool") [[ "${{ inputs.wasm-pack }}" == "true" ]] && mods+=("wasm-pack") - [[ "${{ inputs.markdownlint }}" == "true" ]] && mods+=("markdownlint") + [[ "${{ inputs.markdownlint-cli2 }}" == "true" ]] && mods+=("markdownlint-cli2") [[ "${{ inputs.datadog-ci }}" == "true" ]] && mods+=("datadog-ci") csm=$(IFS=,; echo "${mods[*]}") diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5572b2d3f8d9d..986b145305f78 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,7 +147,7 @@ jobs: - uses: ./.github/actions/setup with: rust: true - markdownlint: true + markdownlint-cli2: true - run: make check-markdown check-generated-docs: diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc index 5bca38386599b..5e442a0162df4 100644 --- a/.markdownlint.jsonc +++ b/.markdownlint.jsonc @@ -1,7 +1,4 @@ -// This config is auto-detected by markdownlint-cli2 (used by Zed, VS Code, and most modern editors). -// The CI check currently uses markdownlint-cli (legacy). Migrate CI to markdownlint-cli2 to unify tooling: -// npm install -g markdownlint-cli2 -// markdownlint-cli2 "**/*.md" +// Auto-detected by markdownlint-cli2 and editors (Zed, VS Code, JetBrains). { "default": true, diff --git a/Makefile b/Makefile index 8b1b856647b9c..5c52c367c11b7 100644 --- a/Makefile +++ b/Makefile @@ -508,7 +508,7 @@ check-markdown: ## Check that markdown is styled properly .PHONY: fix-markdown fix-markdown: ## Auto-fix markdown style issues - ${MAYBE_ENVIRONMENT_EXEC} markdownlint --fix --config .markdownlint.jsonc $(shell git ls-files '*.md') + ${MAYBE_ENVIRONMENT_EXEC} markdownlint-cli2 --fix $(shell git ls-files '*.md') .PHONY: check-examples check-examples: ## Check that the config/examples files are valid diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 5bd5a29a79ac2..397fefbb464a8 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -37,7 +37,7 @@ CARGO_HACK_VERSION="0.6.43" DD_RUST_LICENSE_TOOL_VERSION="1.0.6" CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" -MARKDOWNLINT_VERSION="0.45.0" +MARKDOWNLINT_CLI2_VERSION="0.22.0" DATADOG_CI_VERSION="5.9.0" VDEV_VERSION="0.3.0" @@ -52,7 +52,7 @@ ALL_MODULES=( cargo-llvm-cov dd-rust-license-tool wasm-pack - markdownlint + markdownlint-cli2 datadog-ci release-flags # Not a tool - sources release-flags.sh to set CI env vars vdev @@ -92,7 +92,7 @@ Modules: cargo-llvm-cov dd-rust-license-tool wasm-pack - markdownlint + markdownlint-cli2 datadog-ci vdev @@ -218,5 +218,5 @@ maybe_install_cargo_tool dd-rust-license-tool "${DD_RUST_LICENSE_TOOL_VERSION}" maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" -maybe_install_npm_package markdownlint markdownlint-cli "${MARKDOWNLINT_VERSION}" +maybe_install_npm_package markdownlint-cli2 markdownlint-cli2 "${MARKDOWNLINT_CLI2_VERSION}" maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" diff --git a/vdev/src/commands/check/markdown.rs b/vdev/src/commands/check/markdown.rs index 3ecf0c91f3201..1e10126a14e5e 100644 --- a/vdev/src/commands/check/markdown.rs +++ b/vdev/src/commands/check/markdown.rs @@ -14,11 +14,8 @@ impl Cli { return Ok(()); } - let args: Vec<&str> = vec!["--config", ".markdownlint.jsonc"] - .into_iter() - .chain(files.iter().map(String::as_str)) - .collect(); + let args: Vec<&str> = files.iter().map(String::as_str).collect(); - app::exec("markdownlint", &args, true) + app::exec("markdownlint-cli2", &args, true) } } From 50345d5929ecb45d74e1355e612f044c24ff4a93 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 31 Mar 2026 14:27:15 -0400 Subject: [PATCH 037/364] chore(ci): skip test runner pull, datadog-ci install, and checkout when int tests won't run (#25068) * chore(ci): skip test runner pull, datadog-ci install, and checkout when int tests won't run * chore(ci): evaluate int/e2e run conditions once via step output --- .github/workflows/ci-integration-review.yml | 36 ++++-- .github/workflows/integration.yml | 118 ++++++++++---------- 2 files changed, 80 insertions(+), 74 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 30e359e366f30..72d2b4e660151 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -105,14 +105,19 @@ jobs: fail-fast: false matrix: service: [ - "amqp", "appsignal", "aws", "axiom", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", + "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", + "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", - "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "splunk", "webhdfs" + "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", + "redis", "webhdfs" ] steps: + - name: Evaluate run condition + id: run_condition + run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.run_condition.outputs.should_run == 'true' with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -123,20 +128,21 @@ jobs: datadog-ci: true - name: Pull test runner image + if: steps.run_condition.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - name: Integration Tests - ${{ matrix.service }} - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') - || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }} + if: steps.run_condition.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/run-integration-test.sh int ${{ matrix.service }} + command: | + bash scripts/environment/prepare.sh --modules=datadog-ci + bash scripts/run-integration-test.sh int ${{ matrix.service }} e2e-tests: needs: @@ -154,7 +160,12 @@ jobs: "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ] steps: + - name: Evaluate run condition + id: run_condition + run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.run_condition.outputs.should_run == 'true' with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -165,20 +176,21 @@ jobs: datadog-ci: true - name: Pull test runner image + if: steps.run_condition.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - name: E2E Tests - ${{ matrix.service }} - if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-all') - || startsWith(github.event.review.body, '/ci-run-all') - || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }} + if: steps.run_condition.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/run-integration-test.sh e2e ${{ matrix.service }} + command: | + bash scripts/environment/prepare.sh --modules=datadog-ci + bash scripts/run-integration-test.sh e2e ${{ matrix.service }} update-pr-status: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 20fae17be16ae..861913c315cd4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -84,59 +84,56 @@ jobs: ] timeout-minutes: 90 steps: + - name: Download JSON artifact from changes.yml + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + if: github.event_name == 'merge_group' + with: + name: int_tests_changes + + - name: Determine if test should run + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ needs.changes.outputs.dependencies }}" == "true" || \ + "${{ needs.changes.outputs.integration-yml }}" == "true" ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + elif [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then + echo "should_run=false" >> "$GITHUB_OUTPUT" + elif [[ -f int_tests_changes.json ]]; then + should_run=$(jq -r '."${{ matrix.service }}" // false' int_tests_changes.json) + echo "should_run=${should_run}" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.check.outputs.should_run == 'true' with: submodules: "recursive" - uses: ./.github/actions/setup + if: steps.check.outputs.should_run == 'true' with: vdev: true mold: false cargo-cache: false - - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - if: github.event_name == 'merge_group' - with: - name: int_tests_changes + datadog-ci: true - name: Pull test runner image + if: steps.check.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.sha }} - name: Run Integration Tests for ${{ matrix.service }} + if: steps.check.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 command: | - if [[ -f int_tests_changes.json ]]; then - # Parse the JSON and check if the specific integration test should run. - should_run=$(jq -r '."${{ matrix.service }}" // false' int_tests_changes.json) - else - # The `changes` job did not run (manual run) or the file is missing, default to false. - should_run=false - fi - - if [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then - echo "Skipping ${{ matrix.service }} test since only website changes were detected" - exit 0 - fi - - # Check if any of the three conditions is true - if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ - "${{ needs.changes.outputs.dependencies }}" == "true" || \ - "${{ needs.changes.outputs.integration-yml }}" == "true" || \ - "$should_run" == "true" ]]; then - # Only install dep if test runs - bash scripts/environment/prepare.sh --modules=datadog-ci - echo "Running test for ${{ matrix.service }}" - bash scripts/run-integration-test.sh int ${{ matrix.service }} - else - echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." - fi + bash scripts/run-integration-test.sh int ${{ matrix.service }} e2e-tests: @@ -152,59 +149,56 @@ jobs: timeout-minutes: 90 steps: + - name: Download JSON artifact from changes.yml + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + if: github.event_name == 'merge_group' + with: + name: e2e_tests_changes + + - name: Determine if test should run + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ needs.changes.outputs.dependencies }}" == "true" || \ + "${{ needs.changes.outputs.integration-yml }}" == "true" ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + elif [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then + echo "should_run=false" >> "$GITHUB_OUTPUT" + elif [[ -f e2e_tests_changes.json ]]; then + should_run=$(jq -r '."${{ matrix.service }}" // false' e2e_tests_changes.json) + echo "should_run=${should_run}" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.check.outputs.should_run == 'true' with: submodules: "recursive" - uses: ./.github/actions/setup + if: steps.check.outputs.should_run == 'true' with: vdev: true mold: false cargo-cache: false - - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - if: github.event_name == 'merge_group' - with: - name: e2e_tests_changes + datadog-ci: true - name: Pull test runner image + if: steps.check.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.sha }} - name: Run E2E Tests for ${{ matrix.service }} + if: steps.check.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 command: | - if [[ -f e2e_tests_changes.json ]]; then - # Parse the JSON and check if the specific e2e test should run. - should_run=$(jq -r '."${{ matrix.service }}" // false' e2e_tests_changes.json) - else - # The `changes` job did not run (manual run) or the file is missing, default to false. - should_run=false - fi - - if [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then - echo "Skipping ${{ matrix.service }} test since only website changes were detected" - exit 0 - fi - - # Check if any of the three conditions is true - if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ - "${{ needs.changes.outputs.dependencies }}" == "true" || \ - "${{ needs.changes.outputs.integration-yml }}" == "true" ]] || \ - "$should_run" == "true" ]]; then - # Only install dep if test runs - bash scripts/environment/prepare.sh --modules=datadog-ci - echo "Running test for ${{ matrix.service }}" - bash scripts/run-integration-test.sh e2e ${{ matrix.service }} - else - echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." - fi + bash scripts/run-integration-test.sh e2e ${{ matrix.service }} integration-test-suite: name: Integration Test Suite From 24d8db42e01e28b590e5484fede9279908805901 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 31 Mar 2026 18:04:57 -0400 Subject: [PATCH 038/364] chore(ci): remove unused publish-homebrew workflow (#25085) Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/publish-homebrew.yml | 36 -------------------------- 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/publish-homebrew.yml diff --git a/.github/workflows/publish-homebrew.yml b/.github/workflows/publish-homebrew.yml deleted file mode 100644 index 117fe087c8441..0000000000000 --- a/.github/workflows/publish-homebrew.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Publish to Homebrew - -on: - workflow_dispatch: - inputs: - vector_version: - description: "Vector version to publish" - required: true - type: string - - workflow_call: - inputs: - git_ref: - required: true - type: string - vector_version: - required: true - type: string - -permissions: - contents: read - -jobs: - publish-homebrew: - runs-on: ubuntu-24.04 - timeout-minutes: 10 - env: - GITHUB_TOKEN: ${{ secrets.HOMEBREW_PAT }} - steps: - - name: Checkout Vector - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.git_ref || github.ref_name }} - - - name: Publish update to Homebrew tap - run: make release-homebrew VECTOR_VERSION=${{ inputs.vector_version }} From df00c7eec91ecb35eb0a8db0249701562de7dc87 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 1 Apr 2026 15:37:50 -0400 Subject: [PATCH 039/364] fix(ci): pin Pulsar integration test image to 4.1.3 (#25097) The `apachepulsar/pulsar:latest` image moved to a version that breaks TLS connectivity, causing `pulsar_happy_tls` and `consumes_event_with_tls` to fail consistently in CI. Pin to 4.1.3 to unblock the merge queue. Re-enabling `latest` is tracked in #25096. Co-authored-by: Claude Opus 4.6 (1M context) --- tests/integration/pulsar/config/test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/pulsar/config/test.yaml b/tests/integration/pulsar/config/test.yaml index 9d7f33f60dac5..a273fb4e9f163 100644 --- a/tests/integration/pulsar/config/test.yaml +++ b/tests/integration/pulsar/config/test.yaml @@ -7,7 +7,9 @@ env: PULSAR_HOST: pulsar matrix: - version: [latest] + # TODO: re-enable `latest` once TLS tests pass with newer Pulsar images. + # See https://github.com/vectordotdev/vector/issues/25096 + version: ["4.1.3"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From 639c924c733442bcc7443817d16ea0dd737ded54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:24:01 +0000 Subject: [PATCH 040/364] chore(deps): bump debian from `1d3c811` to `26f98cc` in /distribution/docker/debian in the docker-images group across 1 directory (#24996) chore(deps): bump debian Bumps the docker-images group in /distribution/docker/debian with 1 update: debian. Updates `debian` from `1d3c811` to `26f98cc` --- updated-dependencies: - dependency-name: debian dependency-version: trixie-slim dependency-type: direct:production dependency-group: docker-images ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- distribution/docker/debian/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/docker/debian/Dockerfile b/distribution/docker/debian/Dockerfile index a726f8a72004c..4922e12ef9a60 100644 --- a/distribution/docker/debian/Dockerfile +++ b/distribution/docker/debian/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 AS builder +FROM docker.io/debian:trixie-slim@sha256:26f98ccd92fd0a44d6928ce8ff8f4921b4d2f535bfa07555ee5d18f61429cf0c AS builder WORKDIR /vector @@ -7,7 +7,7 @@ RUN dpkg -i vector_*_"$(dpkg --print-architecture)".deb RUN mkdir -p /var/lib/vector -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 +FROM docker.io/debian:trixie-slim@sha256:26f98ccd92fd0a44d6928ce8ff8f4921b4d2f535bfa07555ee5d18f61429cf0c # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" From 9dbdb0662b668867ac8db6e444ac710d4346a037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:25:08 +0000 Subject: [PATCH 041/364] chore(deps): bump distroless/static from `28efbe9` to `47b2d72` in /distribution/docker/distroless-static in the docker-images group across 1 directory (#24998) chore(deps): bump distroless/static Bumps the docker-images group in /distribution/docker/distroless-static with 1 update: distroless/static. Updates `distroless/static` from `28efbe9` to `47b2d72` --- updated-dependencies: - dependency-name: distroless/static dependency-version: latest dependency-type: direct:production dependency-group: docker-images ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- distribution/docker/distroless-static/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/distroless-static/Dockerfile b/distribution/docker/distroless-static/Dockerfile index 9518be0c4492c..312190cda5302 100644 --- a/distribution/docker/distroless-static/Dockerfile +++ b/distribution/docker/distroless-static/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /var/lib/vector # distroless doesn't use static tags # hadolint ignore=DL3007 -FROM gcr.io/distroless/static:latest@sha256:28efbe90d0b2f2a3ee465cc5b44f3f2cf5533514cf4d51447a977a5dc8e526d0 +FROM gcr.io/distroless/static:latest@sha256:47b2d72ff90843eb8a768b5c2f89b40741843b639d065b9b937b07cd59b479c6 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" From 3c879836d586695701e9cf60fed3e7ef4b408f2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:25:30 +0000 Subject: [PATCH 042/364] chore(deps): bump debian from `1d3c811` to `26f98cc` in /distribution/docker/distroless-libc in the docker-images group across 1 directory (#24997) chore(deps): bump debian Bumps the docker-images group in /distribution/docker/distroless-libc with 1 update: debian. Updates `debian` from `1d3c811` to `26f98cc` --- updated-dependencies: - dependency-name: debian dependency-version: trixie-slim dependency-type: direct:production dependency-group: docker-images ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- distribution/docker/distroless-libc/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/distroless-libc/Dockerfile b/distribution/docker/distroless-libc/Dockerfile index 2e3988b7403fe..0ae19e00e17d8 100644 --- a/distribution/docker/distroless-libc/Dockerfile +++ b/distribution/docker/distroless-libc/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 AS builder +FROM docker.io/debian:trixie-slim@sha256:26f98ccd92fd0a44d6928ce8ff8f4921b4d2f535bfa07555ee5d18f61429cf0c AS builder WORKDIR /vector From 9f484c2b9c1c219f44c2fd5289c40e3a4b63f6e3 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 19:26:12 -0400 Subject: [PATCH 043/364] fix(ci): prevent script injection in integration review workflow (#25106) --- .github/workflows/ci-integration-review.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 72d2b4e660151..029ea40e05963 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -114,7 +114,15 @@ jobs: steps: - name: Evaluate run condition id: run_condition - run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + env: + REVIEW_BODY: ${{ github.event.review.body }} + SERVICE: ${{ matrix.service }} + run: | + if [[ "$REVIEW_BODY" == /ci-run-integration-all* ]] || [[ "$REVIEW_BODY" == /ci-run-all* ]] || [[ "$REVIEW_BODY" == "/ci-run-integration-${SERVICE}"* ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: steps.run_condition.outputs.should_run == 'true' @@ -162,7 +170,15 @@ jobs: steps: - name: Evaluate run condition id: run_condition - run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + env: + REVIEW_BODY: ${{ github.event.review.body }} + SERVICE: ${{ matrix.service }} + run: | + if [[ "$REVIEW_BODY" == /ci-run-e2e-all* ]] || [[ "$REVIEW_BODY" == /ci-run-all* ]] || [[ "$REVIEW_BODY" == "/ci-run-e2e-${SERVICE}"* ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: steps.run_condition.outputs.should_run == 'true' From 9055388bc0c89a56177f6df60de7a7e09fe74de1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 2 Apr 2026 09:48:09 -0400 Subject: [PATCH 044/364] chore(dev): add repo-wide prettier config for YAML, JS, TS, and JSON (#25082) * chore: add repo-wide prettier for YAML/JS/TS/JSON formatting Add a root .prettierrc.json (auto-detected by Zed, VS Code, JetBrains) and .prettierignore to skip Hugo templates, generated files, and VRL configs with triple-quoted strings. - Pin prettier@3.5.3 in prepare.sh and .github/actions/setup - Integrate prettier into vdev check fmt / vdev fmt - Add check-prettier and fix-prettier Makefile targets - Add prettier path filter in changes.yml so check-fmt triggers on YAML/JS/TS/JSON changes - Remove website/.prettierrc.json (superseded by root config) Co-Authored-By: Claude Opus 4.6 (1M context) * chore: format all YAML/JS/TS/JSON files with prettier Run make fix-prettier to establish a consistent formatting baseline. Changes are purely cosmetic: whitespace cleanup, quote normalization, and array formatting. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: use git ls-files instead of globs for prettier file discovery Shell globs expand to include untracked directories. Use git ls-files to only format tracked files, consistent with how markdownlint works. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: pass --ignore-path .prettierignore when using git ls-files When prettier receives explicit file paths it skips .prettierignore. Pass --ignore-path explicitly so exclusions (lib/codecs/tests/data, config examples with VRL triple-quotes, Hugo build output, etc.) are still respected. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: suppress file list echo in prettier Makefile targets Co-Authored-By: Claude Opus 4.6 (1M context) * chore: invoke prettier per file extension to avoid ARG_MAX Running prettier with all tracked files in a single invocation can exceed macOS ARG_MAX (262 KB) since this repo has ~4400 matching paths. Invoke prettier once per extension type instead. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: log per-extension prettier invocations in vdev Move info! inside the loop so each extension type logs its own progress line. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: bump prettier to 3.8.1 to match local version Co-Authored-By: Claude Opus 4.6 (1M context) * chore: address PR feedback on prettier setup - Remove unused vendor/ entry from .prettierignore - Deduplicate PRETTIER_EXTENSIONS constant between fmt and check/fmt Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/DISCUSSION_TEMPLATE/q-a.yml | 2 +- .github/ISSUE_TEMPLATE/bug.yml | 152 +-- .github/ISSUE_TEMPLATE/config.yml | 1 - .github/ISSUE_TEMPLATE/feature.yml | 167 ++- .github/actions/install-vdev/action.yml | 2 +- .github/actions/pull-test-runner/action.yml | 4 +- .github/actions/setup/action.yml | 7 +- .github/dependabot.yml | 52 +- .github/labeler.yml | 24 +- .github/workflows/build-test-runner.yml | 8 +- .github/workflows/build_preview_sites.yml | 32 +- .github/workflows/changes.yml | 222 +-- .github/workflows/ci-integration-review.yml | 67 +- .github/workflows/ci-review-trigger.yml | 28 +- .github/workflows/component_features.yml | 6 +- .github/workflows/coverage.yml | 4 +- .github/workflows/create_preview_sites.yml | 12 +- .github/workflows/cross.yml | 2 +- .github/workflows/custom_builds.yml | 6 +- .github/workflows/deny.yml | 6 +- .github/workflows/environment.yml | 6 +- .github/workflows/install-sh.yml | 4 +- .github/workflows/integration-test.yml | 2 +- .github/workflows/integration.yml | 51 +- .github/workflows/k8s_e2e.yml | 6 +- .github/workflows/labeler.yml | 8 +- .github/workflows/nightly.yml | 8 +- .github/workflows/preview_site_trigger.yml | 6 +- .github/workflows/protobuf.yml | 10 +- .github/workflows/publish.yml | 16 +- .github/workflows/regression.yml | 27 +- .github/workflows/release.yml | 6 +- .github/workflows/scorecard.yml | 4 +- .github/workflows/spelling.yml | 105 +- .github/workflows/static-analysis.yml | 4 +- .github/workflows/test-make-command.yml | 11 +- .github/workflows/test.yml | 5 +- .github/workflows/unit_mac.yml | 3 +- .github/workflows/unit_windows.yml | 3 +- .github/workflows/vdev_publish.yml | 6 +- .prettierignore | 30 + .prettierrc.json | 4 + Makefile | 18 + config/examples/environment_variables.yaml | 6 +- .../examples/file_to_cloudwatch_metrics.yaml | 12 +- .../namespacing/sinks/es_cluster.yaml | 6 +- .../namespacing/sinks/s3_archives.yaml | 12 +- .../namespacing/transforms/apache_sample.yaml | 2 +- config/examples/stdio.yaml | 2 +- config/examples/wrapped_json.yaml | 6 +- .../collector/logs/v1/logs_service_http.yaml | 9 +- .../metrics/v1/metrics_service_http.yaml | 9 +- .../trace/v1/trace_service_http.yaml | 8 +- .../data/fixtures/log_event/malformed.json | 2 +- .../resources/json-schema_definition.json | 11 +- lib/vector-vrl/web-playground/public/index.js | 552 ++++---- .../web-playground/public/vrl-highlighter.js | 1199 ++++++++--------- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 36 +- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 36 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 30 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 30 +- .../file_100_to_blackhole/lading/lading.yaml | 37 +- .../file_100_to_blackhole/vector/vector.yaml | 10 +- .../file_to_blackhole/lading/lading.yaml | 37 +- .../file_to_blackhole/vector/vector.yaml | 10 +- .../fluent_elasticsearch/lading/lading.yaml | 37 +- .../fluent_elasticsearch/vector/vector.yaml | 16 +- .../http_elasticsearch/lading/lading.yaml | 37 +- .../http_elasticsearch/vector/vector.yaml | 16 +- .../http_text_to_http_json/lading/lading.yaml | 37 +- .../http_text_to_http_json/vector/vector.yaml | 10 +- .../http_to_http_acks/lading/lading.yaml | 37 +- .../http_to_http_acks/vector/vector.yaml | 16 +- .../http_to_http_disk_buffer/experiment.yaml | 1 - .../lading/lading.yaml | 38 +- .../vector/vector.yaml | 12 +- .../http_to_http_json/lading/lading.yaml | 37 +- .../http_to_http_json/vector/vector.yaml | 16 +- .../http_to_http_noack/lading/lading.yaml | 37 +- .../http_to_http_noack/vector/vector.yaml | 16 +- .../cases/http_to_s3/lading/lading.yaml | 36 +- .../cases/http_to_s3/vector/vector.yaml | 18 +- .../otlp_grpc_to_blackhole/lading/lading.yaml | 37 +- .../otlp_grpc_to_blackhole/vector/vector.yaml | 10 +- .../otlp_http_to_blackhole/lading/lading.yaml | 37 +- .../otlp_http_to_blackhole/vector/vector.yaml | 10 +- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 14 +- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 10 +- .../splunk_hec_route_s3/lading/lading.yaml | 36 +- .../splunk_hec_route_s3/vector/vector.yaml | 36 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 14 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 14 +- .../lading/lading.yaml | 36 +- .../syslog_humio_logs/lading/lading.yaml | 36 +- .../syslog_humio_logs/vector/vector.yaml | 16 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 20 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 20 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 22 +- .../cases/syslog_loki/lading/lading.yaml | 37 +- .../cases/syslog_loki/vector/vector.yaml | 16 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 26 +- .../syslog_splunk_hec_logs/lading/lading.yaml | 36 +- .../syslog_splunk_hec_logs/vector/vector.yaml | 16 +- regression/config.yaml | 1 - scripts/environment/prepare.sh | 4 + tests/data/cmd/config/config_3.json | 8 +- tests/data/secret-backends/file-secrets.json | 2 +- tests/e2e/datadog-logs/config/compose.yaml | 22 +- tests/e2e/datadog-logs/config/test.yaml | 25 +- tests/e2e/datadog-logs/data/agent_only.yaml | 4 +- tests/e2e/datadog-logs/data/agent_vector.yaml | 4 +- tests/e2e/datadog-metrics/config/compose.yaml | 15 +- tests/e2e/datadog-metrics/config/test.yaml | 26 +- .../e2e/datadog-metrics/data/agent_only.yaml | 2 +- .../datadog-metrics/data/agent_vector.yaml | 2 +- .../opentelemetry-logs/config/compose.yaml | 6 +- tests/e2e/opentelemetry-logs/config/test.yaml | 10 +- .../data/collector-sink.yaml | 10 +- .../data/collector-source.yaml | 16 +- .../data/vector_default.yaml | 3 +- .../opentelemetry-metrics/config/compose.yaml | 6 +- .../opentelemetry-metrics/config/test.yaml | 8 +- .../data/collector-sink.yaml | 10 +- .../data/collector-source.yaml | 10 +- tests/integration/amqp/config/compose.yaml | 2 +- tests/integration/amqp/config/test.yaml | 18 +- tests/integration/appsignal/config/test.yaml | 8 +- tests/integration/aws/config/compose.yaml | 12 +- tests/integration/aws/config/test.yaml | 18 +- tests/integration/axiom/config/test.yaml | 10 +- tests/integration/azure/config/compose.yaml | 4 +- tests/integration/azure/config/test.yaml | 10 +- .../clickhouse/config/compose.yaml | 2 +- tests/integration/clickhouse/config/test.yaml | 12 +- .../integration/databend/config/compose.yaml | 2 +- tests/integration/databend/config/test.yaml | 12 +- .../datadog-agent/config/compose.yaml | 40 +- .../datadog-agent/config/test.yaml | 14 +- .../integration/datadog-logs/config/test.yaml | 14 +- .../datadog-metrics/config/test.yaml | 14 +- .../datadog-traces/config/compose.yaml | 46 +- .../datadog-traces/config/test.yaml | 18 +- tests/integration/dnstap/config/test.yaml | 14 +- .../integration/docker-logs/config/test.yaml | 12 +- tests/integration/doris/config/test.yaml | 16 +- .../elasticsearch/config/compose.yaml | 30 +- .../elasticsearch/config/test.yaml | 10 +- .../eventstoredb/config/compose.yaml | 4 +- .../integration/eventstoredb/config/test.yaml | 12 +- tests/integration/fluent/config/test.yaml | 10 +- tests/integration/gcp/config/compose.yaml | 14 +- tests/integration/gcp/config/invalidauth.json | 2 +- tests/integration/gcp/config/test.yaml | 20 +- .../greptimedb/config/compose.yaml | 2 +- tests/integration/greptimedb/config/test.yaml | 8 +- .../http-client/config/compose.yaml | 34 +- .../integration/http-client/config/test.yaml | 10 +- .../http-client/data/serve/logs/json.json | 2 +- .../data/serve/metrics/native.json | 2 +- .../http-client/data/serve/traces/native.json | 2 +- tests/integration/humio/config/compose.yaml | 2 +- tests/integration/humio/config/test.yaml | 10 +- .../integration/influxdb/config/compose.yaml | 16 +- tests/integration/influxdb/config/test.yaml | 14 +- tests/integration/kafka/config/compose.yaml | 48 +- tests/integration/kafka/config/test.yaml | 18 +- .../integration/logstash/config/compose.yaml | 10 +- tests/integration/logstash/config/test.yaml | 10 +- tests/integration/logstash/data/heartbeat.yml | 10 +- tests/integration/loki/config/compose.yaml | 2 +- tests/integration/loki/config/test.yaml | 12 +- tests/integration/mongodb/config/compose.yaml | 38 +- tests/integration/mongodb/config/test.yaml | 12 +- tests/integration/mqtt/config/compose.yaml | 4 +- tests/integration/mqtt/config/test.yaml | 14 +- tests/integration/nats/config/compose.yaml | 44 +- tests/integration/nginx/config/compose.yaml | 16 +- tests/integration/nginx/config/test.yaml | 12 +- .../opentelemetry/config/compose.yaml | 4 +- .../opentelemetry/config/test.yaml | 10 +- .../opentelemetry/data/config.yaml | 6 +- .../integration/postgres/config/compose.yaml | 12 +- tests/integration/postgres/config/test.yaml | 16 +- .../prometheus/config/compose.yaml | 20 +- tests/integration/prometheus/config/test.yaml | 20 +- .../prometheus/data/prometheus.yaml | 8 +- tests/integration/pulsar/config/compose.yaml | 2 +- tests/integration/pulsar/config/test.yaml | 12 +- tests/integration/redis/config/compose.yaml | 2 +- tests/integration/redis/config/test.yaml | 14 +- .../integration/shutdown/config/compose.yaml | 48 +- tests/integration/shutdown/config/test.yaml | 6 +- tests/integration/splunk/config/compose.yaml | 16 +- tests/integration/splunk/config/test.yaml | 16 +- tests/integration/webhdfs/config/compose.yaml | 2 +- tests/integration/webhdfs/config/test.yaml | 12 +- .../windows-event-log/config/test.yaml | 10 +- .../ignore-invalid/sinks/es_cluster.json | 6 +- .../namespacing/success/sinks/es_cluster.json | 6 +- vdev/src/commands/check/fmt.rs | 19 +- vdev/src/commands/fmt.rs | 22 +- website/.htmltest.external.yml | 16 +- website/.prettierrc.json | 17 - website/algolia.json | 33 +- website/assets/js/below.js | 16 +- website/assets/js/home.tsx | 346 +++-- website/assets/js/search.tsx | 209 ++- website/babel.config.js | 18 +- website/data/redirects.yaml | 132 +- website/postcss.config.js | 16 +- website/scripts/create-config-examples.js | 143 +- website/scripts/typesense-index.ts | 50 +- website/scripts/typesense-sync.ts | 13 +- website/typesense.config.json | 2 +- 226 files changed, 4028 insertions(+), 2779 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc.json delete mode 100644 website/.prettierrc.json diff --git a/.github/DISCUSSION_TEMPLATE/q-a.yml b/.github/DISCUSSION_TEMPLATE/q-a.yml index 23cb1f8467745..cc4942b0976ce 100644 --- a/.github/DISCUSSION_TEMPLATE/q-a.yml +++ b/.github/DISCUSSION_TEMPLATE/q-a.yml @@ -1,5 +1,5 @@ title: "Q&A" -labels: [ q-a ] +labels: [q-a] body: - type: markdown attributes: diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 47354ec1ba30e..ff1c7df5e83ca 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -1,94 +1,94 @@ name: Bug description: 🐛 Let us know about an unexpected error, a crash, or an incorrect behavior. -type: 'Bug' +type: "Bug" body: -- type: markdown - attributes: - value: | - Thank you for opening 🐛 bug report! + - type: markdown + attributes: + value: | + Thank you for opening 🐛 bug report! -- type: textarea - attributes: - label: A note for the community - value: | - - * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request - * If you are interested in working on this issue or have submitted a pull request, please leave a comment - + - type: textarea + attributes: + label: A note for the community + value: | + + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + -- type: textarea - id: problem - attributes: - label: Problem - description: > - Please provide a clear and concise description of what the bug is, - including what currently happens and what you expected to happen. - validations: - required: true + - type: textarea + id: problem + attributes: + label: Problem + description: > + Please provide a clear and concise description of what the bug is, + including what currently happens and what you expected to happen. + validations: + required: true -- type: textarea - id: config - attributes: - label: Configuration - description: | - Paste the relevant parts of your Vector configuration file. + - type: textarea + id: config + attributes: + label: Configuration + description: | + Paste the relevant parts of your Vector configuration file. - !! If your config files contain sensitive information please remove it !! - render: text + !! If your config files contain sensitive information please remove it !! + render: text -- type: input - id: version - attributes: - label: Version - description: | - Please paste the output of running `vector --version`. + - type: input + id: version + attributes: + label: Version + description: | + Please paste the output of running `vector --version`. - If you are not running the latest version of Vector, please try upgrading - because your issue may have already been fixed. - validations: - required: true + If you are not running the latest version of Vector, please try upgrading + because your issue may have already been fixed. + validations: + required: true -- type: textarea - id: debug - attributes: - label: Debug Output - description: | - Full debug output can be obtained by running Vector with the following: + - type: textarea + id: debug + attributes: + label: Debug Output + description: | + Full debug output can be obtained by running Vector with the following: - ``` - RUST_BACKTRACE=full vector -vvv - ``` + ``` + RUST_BACKTRACE=full vector -vvv + ``` - If the debug output is long, please create a GitHub Gist containing the debug output and paste the link here. + If the debug output is long, please create a GitHub Gist containing the debug output and paste the link here. - !! Debug output may contain sensitive information. Please review it before posting publicly. !! - render: text + !! Debug output may contain sensitive information. Please review it before posting publicly. !! + render: text -- type: textarea - id: data - attributes: - label: Example Data - description: | - Please provide any example data that will help debug the issue, for example: + - type: textarea + id: data + attributes: + label: Example Data + description: | + Please provide any example data that will help debug the issue, for example: - ``` - 201.69.207.46 - kemmer6752 [07/06/2019:14:53:55 -0400] "PATCH /innovative/interfaces" 301 669 - ``` + ``` + 201.69.207.46 - kemmer6752 [07/06/2019:14:53:55 -0400] "PATCH /innovative/interfaces" 301 669 + ``` -- type: textarea - id: context - attributes: - label: Additional Context - description: | - Is there anything atypical about your situation that we should know? For - example: is Vector running in Kubernetes? Are you passing any unusual command - line options or environment variables to opt-in to non-default behavior? + - type: textarea + id: context + attributes: + label: Additional Context + description: | + Is there anything atypical about your situation that we should know? For + example: is Vector running in Kubernetes? Are you passing any unusual command + line options or environment variables to opt-in to non-default behavior? -- type: textarea - id: references - attributes: - label: References - description: | - Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: + - type: textarea + id: references + attributes: + label: References + description: | + Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: - - #6017 + - #6017 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index ea44ff1b06562..df12e12af4f76 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -12,4 +12,3 @@ contact_links: - name: Twitter url: https://twitter.com/vectordotdev about: Follow us and stay up to date with Vector. - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index f2e066bb37303..8b7a6fc139185 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -1,87 +1,86 @@ name: Feature -type: 'Feature' +type: "Feature" description: 🚀 Suggest a new feature. body: -- type: markdown - attributes: - value: | - Thank you for opening 🚀 feature request! - - For general questions about Vector usage, please see - https://discussions.vector.dev. - -- type: textarea - attributes: - label: A note for the community - value: | - - * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request - * If you are interested in working on this issue or have submitted a pull request, please leave a comment - - -- type: textarea - id: use-cases - attributes: - label: Use Cases - description: | - In order to properly evaluate a feature request, it is necessary to - understand the use-cases for it. - - Please describe below the _end goal_ you are trying to achieve that has - led you to request this feature. - - Please keep this section focused on the problem and not on the suggested - solution. We'll get to that in a moment, below! - - -- type: textarea - id: attempted-solutions - attributes: - label: Attempted Solutions - description: | - If you've already tried to solve the problem within Vector's existing - features and found a limitation that prevented you from succeeding, please - describe it below in as much detail as possible. - - Ideally, this would include real configuration snippets that you tried - and what results you got in each case. - - Please remove any sensitive information such as passwords before sharing - configuration snippets and command lines. - -- type: textarea - id: proposal - attributes: - label: Proposal - description: | - If you have an idea for a way to address the problem via a change to - Vector features, please describe it below. - - In this section, it's helpful to include specific examples of how what - you are suggesting might look in configuration files, or on the command line, - since that allows us to understand the full picture of what you are proposing. - - If you're not sure of some details, don't worry! When we evaluate the - feature request we may suggest modifications as necessary to work within the - design constraints of Vector. - -- type: textarea - id: references - attributes: - label: References - description: | - Are there any other GitHub issues, whether open or closed, that are - related to the problem you've described above or to the suggested solution? If - so, please create a list below that mentions each of them. For example: - - - #7023 - -- type: input - id: version - attributes: - label: Version - description: | - Please paste the output of running `vector --version`. - - This will record which version was current at the time of your feature request, - to help manage the request backlog. + - type: markdown + attributes: + value: | + Thank you for opening 🚀 feature request! + + For general questions about Vector usage, please see + https://discussions.vector.dev. + + - type: textarea + attributes: + label: A note for the community + value: | + + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + + + - type: textarea + id: use-cases + attributes: + label: Use Cases + description: | + In order to properly evaluate a feature request, it is necessary to + understand the use-cases for it. + + Please describe below the _end goal_ you are trying to achieve that has + led you to request this feature. + + Please keep this section focused on the problem and not on the suggested + solution. We'll get to that in a moment, below! + + - type: textarea + id: attempted-solutions + attributes: + label: Attempted Solutions + description: | + If you've already tried to solve the problem within Vector's existing + features and found a limitation that prevented you from succeeding, please + describe it below in as much detail as possible. + + Ideally, this would include real configuration snippets that you tried + and what results you got in each case. + + Please remove any sensitive information such as passwords before sharing + configuration snippets and command lines. + + - type: textarea + id: proposal + attributes: + label: Proposal + description: | + If you have an idea for a way to address the problem via a change to + Vector features, please describe it below. + + In this section, it's helpful to include specific examples of how what + you are suggesting might look in configuration files, or on the command line, + since that allows us to understand the full picture of what you are proposing. + + If you're not sure of some details, don't worry! When we evaluate the + feature request we may suggest modifications as necessary to work within the + design constraints of Vector. + + - type: textarea + id: references + attributes: + label: References + description: | + Are there any other GitHub issues, whether open or closed, that are + related to the problem you've described above or to the suggested solution? If + so, please create a list below that mentions each of them. For example: + + - #7023 + + - type: input + id: version + attributes: + label: Version + description: | + Please paste the output of running `vector --version`. + + This will record which version was current at the time of your feature request, + to help manage the request backlog. diff --git a/.github/actions/install-vdev/action.yml b/.github/actions/install-vdev/action.yml index da04417f67358..e98d9731f2a5d 100644 --- a/.github/actions/install-vdev/action.yml +++ b/.github/actions/install-vdev/action.yml @@ -9,7 +9,7 @@ inputs: skip-cache: description: "Skip cache lookup and force compilation" required: false - default: 'false' + default: "false" runs: using: "composite" diff --git a/.github/actions/pull-test-runner/action.yml b/.github/actions/pull-test-runner/action.yml index b46a47fde93e8..8cded209f8a6f 100644 --- a/.github/actions/pull-test-runner/action.yml +++ b/.github/actions/pull-test-runner/action.yml @@ -3,10 +3,10 @@ description: Login to GHCR and pull the pre-built test runner image for integrat inputs: github_token: - description: 'GitHub token for GHCR authentication' + description: "GitHub token for GHCR authentication" required: true commit_sha: - description: 'Commit SHA used to tag the test runner image' + description: "Commit SHA used to tag the test runner image" required: true runs: diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index f9037aebb68e8..fde57a4b33303 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -80,6 +80,10 @@ inputs: required: false default: false description: "Install markdownlint-cli2 (npm)." + prettier: + required: false + default: false + description: "Install prettier (npm)." datadog-ci: required: false default: false @@ -140,7 +144,6 @@ runs: echo "CARGO_TERM_COLOR=always" >> "$GITHUB_ENV" fi - - name: Enable rust matcher if: ${{ env.NEEDS_RUST == 'true' }} run: echo "::add-matcher::.github/matchers/rust.json" @@ -268,6 +271,7 @@ runs: ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack /usr/local/bin/markdownlint-cli2 + /usr/local/bin/prettier /usr/local/bin/datadog-ci key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} restore-keys: | @@ -288,6 +292,7 @@ runs: [[ "${{ inputs.dd-rust-license-tool }}" == "true" ]] && mods+=("dd-rust-license-tool") [[ "${{ inputs.wasm-pack }}" == "true" ]] && mods+=("wasm-pack") [[ "${{ inputs.markdownlint-cli2 }}" == "true" ]] && mods+=("markdownlint-cli2") + [[ "${{ inputs.prettier }}" == "true" ]] && mods+=("prettier") [[ "${{ inputs.datadog-ci }}" == "true" ]] && mods+=("datadog-ci") csm=$(IFS=,; echo "${mods[*]}") diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fd32d14deed15..609da972baa98 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,51 +15,51 @@ updates: patches: applies-to: version-updates patterns: - - "*" + - "*" update-types: - - "patch" + - "patch" amq: patterns: - - "amq-*" + - "amq-*" aws: patterns: - - "aws-*" + - "aws-*" aws-security: applies-to: security-updates patterns: - - "aws-*" + - "aws-*" azure: patterns: - - "azure_*" + - "azure_*" clap: patterns: - - "clap*" + - "clap*" crossbeam: patterns: - - "crossbeam*" + - "crossbeam*" csv: patterns: - - "csv*" + - "csv*" futures: patterns: - - "futures" - - "futures-util" + - "futures" + - "futures-util" metrics: patterns: - - "metrics" - - "metrics-tracing-context" - - "metrics-util" + - "metrics" + - "metrics-tracing-context" + - "metrics-util" netlink: patterns: - - "netlink-*" + - "netlink-*" phf: patterns: - - "phf*" + - "phf*" prost: patterns: - - "prost" - - "prost-*" + - "prost" + - "prost-*" serde: patterns: - "serde" @@ -70,22 +70,22 @@ updates: - "tokio-*" tonic: patterns: - - "tonic" - - "tonic-*" + - "tonic" + - "tonic-*" tower: patterns: - - "tower" - - "tower-*" + - "tower" + - "tower-*" tracing: patterns: - "tracing" - "tracing-*" wasm-bindgen: patterns: - - "wasm-bindgen-*" + - "wasm-bindgen-*" zstd: patterns: - - "zstd*" + - "zstd*" - package-ecosystem: "docker" directory: "/distribution/docker/alpine" schedule: @@ -158,8 +158,8 @@ updates: groups: artifact: patterns: - - "actions/download-artifact" - - "actions/upload-artifact" + - "actions/download-artifact" + - "actions/upload-artifact" - package-ecosystem: "npm" directory: "website/" schedule: diff --git a/.github/labeler.yml b/.github/labeler.yml index b5fe9e5e4eede..687dea74ce164 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,48 +1,48 @@ # Generic component-level labels and high-level groupings. "domain: sources": - changed-files: - - any-glob-to-any-file: ["src/sources/**/*"] + - any-glob-to-any-file: ["src/sources/**/*"] "domain: sinks": - changed-files: - - any-glob-to-any-file: ["src/sinks/**/*"] + - any-glob-to-any-file: ["src/sinks/**/*"] "domain: transforms": - changed-files: - - any-glob-to-any-file: ["src/transforms/**/*"] + - any-glob-to-any-file: ["src/transforms/**/*"] "domain: topology": - changed-files: - - any-glob-to-any-file: ["src/topology/**/*"] + - any-glob-to-any-file: ["src/topology/**/*"] "domain: codecs": - changed-files: - - any-glob-to-any-file: ["src/codecs/**/*"] + - any-glob-to-any-file: ["src/codecs/**/*"] "domain: core": - changed-files: - - any-glob-to-any-file: ["lib/vector-core/**/*"] + - any-glob-to-any-file: ["lib/vector-core/**/*"] "domain: vrl": - changed-files: - - any-glob-to-any-file: ["lib/vrl/**/*"] + - any-glob-to-any-file: ["lib/vrl/**/*"] "domain: ci": - changed-files: - - any-glob-to-any-file: [".github/workflows/*", ".github/actions/**/*", ".github/*.yml"] + - any-glob-to-any-file: [".github/workflows/*", ".github/actions/**/*", ".github/*.yml"] "domain: vdev": - changed-files: - - any-glob-to-any-file: ["vdev/**/*"] + - any-glob-to-any-file: ["vdev/**/*"] "domain: releasing": - changed-files: - - any-glob-to-any-file: ["distribution/**/*", "scripts/package-*", "scripts/release-*"] + - any-glob-to-any-file: ["distribution/**/*", "scripts/package-*", "scripts/release-*"] "domain: rfc": - changed-files: - - any-glob-to-any-file: ["rfcs/**/*"] + - any-glob-to-any-file: ["rfcs/**/*"] "domain: external docs": - changed-files: - - any-glob-to-any-file: ["website/cue/**/*"] + - any-glob-to-any-file: ["website/cue/**/*"] diff --git a/.github/workflows/build-test-runner.yml b/.github/workflows/build-test-runner.yml index 7a7596651ed2c..ca8e51b0b6944 100644 --- a/.github/workflows/build-test-runner.yml +++ b/.github/workflows/build-test-runner.yml @@ -9,11 +9,11 @@ on: workflow_call: inputs: commit_sha: - description: 'Commit SHA to use for image tag' + description: "Commit SHA to use for image tag" required: true type: string checkout_ref: - description: 'Git ref to checkout (defaults to commit_sha)' + description: "Git ref to checkout (defaults to commit_sha)" required: false type: string @@ -24,8 +24,8 @@ jobs: build: runs-on: ubuntu-24.04 permissions: - contents: read # Required by actions/checkout - packages: write # Required to push test runner image to GitHub Container Registry + contents: read # Required by actions/checkout + packages: write # Required to push test runner image to GitHub Container Registry steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/build_preview_sites.yml b/.github/workflows/build_preview_sites.yml index 9df10ce781379..829571deda5da 100644 --- a/.github/workflows/build_preview_sites.yml +++ b/.github/workflows/build_preview_sites.yml @@ -7,17 +7,17 @@ on: - completed permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: deploy_vector_preview_site: if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} permissions: - contents: read # Required by the reusable workflow - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required by the reusable workflow + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d1a7j77663uxsc" @@ -30,11 +30,11 @@ jobs: deploy_rust_doc_preview_site: if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} permissions: - contents: read # Required by the reusable workflow - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required by the reusable workflow + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d1hoyoksbulg25" @@ -47,11 +47,11 @@ jobs: deploy_vrl_playground_preview_site: if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} permissions: - contents: read # Required by the reusable workflow - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required by the reusable workflow + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d2lr4eds605rpz" diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 25599942b02c9..999548111e9a7 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -48,6 +48,8 @@ on: value: ${{ jobs.source.outputs.website_only }} install: value: ${{ jobs.source.outputs.install }} + prettier: + value: ${{ jobs.source.outputs.prettier }} test-yml: value: ${{ jobs.source.outputs.test-yml }} integration-yml: @@ -152,7 +154,7 @@ on: # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code env: BASE_SHA: ${{ inputs.base_ref || (github.event_name == 'merge_group' && github.event.merge_group.base_sha) || github.event.pull_request.base.sha }} @@ -176,6 +178,7 @@ jobs: install: ${{ steps.filter.outputs.install }} k8s: ${{ steps.filter.outputs.k8s }} website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }} + prettier: ${{ steps.filter.outputs.prettier }} test-yml: ${{ steps.filter.outputs.test-yml }} integration-yml: ${{ steps.filter.outputs.integration-yml }} test-make-command-yml: ${{ steps.filter.outputs.test-make-command-yml }} @@ -184,109 +187,118 @@ jobs: unit_mac-yml: ${{ steps.filter.outputs.unit_mac-yml }} unit_windows-yml: ${{ steps.filter.outputs.unit_windows-yml }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: filter - with: - base: ${{ env.BASE_SHA }} - ref: ${{ env.HEAD_SHA }} - filters: | - source: - - ".cargo/**" - - "benches/**" - - "lib/**" - - "proto/**" - - "scripts/**" - - "src/**" - - "tests/**" - - "build.rs" - - "Cargo.lock" - - "Cargo.toml" - - "Makefile" - - "rust-toolchain.toml" - - "vdev/**" - - ".github/workflows/changes.yml" - dependencies: - - ".cargo/**" - - 'Cargo.toml' - - 'Cargo.lock' - - 'rust-toolchain.toml' - - 'Makefile' - - 'scripts/cross/**' - - "vdev/**" - - ".github/workflows/changes.yml" - deny: - - '**/Cargo.toml' - - 'Cargo.lock' - - ".github/workflows/deny.yml" - cue: - - 'website/cue/**' - - "vdev/**" - - ".github/workflows/changes.yml" - component_docs: - - 'scripts/generate-component-docs.rb' - - "vdev/**" - - 'website/cue/**/*.cue' - - 'docs/generated/**' - # If changes to the VRL sha is made the combined generated cue file will change which - # may cause issues - - 'Cargo.lock' - - ".github/workflows/changes.yml" - markdown: - - '**/**.md' - - "vdev/**" - - ".github/workflows/changes.yml" - scripts: - - '**/**.sh' - - ".github/workflows/changes.yml" - internal_events: - - 'src/internal_events/**' - - "vdev/**" - - ".github/workflows/changes.yml" - docker: - - 'distribution/docker/**' - - "vdev/**" - - ".github/workflows/changes.yml" - install: - - ".github/workflows/install-sh.yml" - - "distribution/install.sh" - - ".github/workflows/changes.yml" - k8s: - - "src/sources/kubernetes_logs/**" - - "lib/k8s-e2e-tests/**" - - "lib/k8s-test-framework/**" - - "scripts/test-e2e-kubernetes.sh" - - "scripts/build-docker.sh" - - "scripts/deploy-chart-test.sh" - - ".github/workflows/k8s_e2e.yml" - - ".github/workflows/changes.yml" - website: - - "website/**" - not_website: - - "!website/**" - test-yml: - - ".github/workflows/test.yml" - - ".github/workflows/changes.yml" - integration-yml: - - ".github/workflows/integration.yml" - - ".github/workflows/changes.yml" - test-make-command-yml: - - ".github/workflows/test-make-command.yml" - - ".github/workflows/changes.yml" - msrv-yml: - - ".github/workflows/msrv.yml" - - ".github/workflows/changes.yml" - cross-yml: - - ".github/workflows/cross.yml" - - ".github/workflows/changes.yml" - unit_mac-yml: - - ".github/workflows/unit_mac.yml" - - ".github/workflows/changes.yml" - unit_windows-yml: - - ".github/workflows/unit_windows.yml" - - ".github/workflows/changes.yml" + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} + filters: | + source: + - ".cargo/**" + - "benches/**" + - "lib/**" + - "proto/**" + - "scripts/**" + - "src/**" + - "tests/**" + - "build.rs" + - "Cargo.lock" + - "Cargo.toml" + - "Makefile" + - "rust-toolchain.toml" + - "vdev/**" + - ".github/workflows/changes.yml" + dependencies: + - ".cargo/**" + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - 'Makefile' + - 'scripts/cross/**' + - "vdev/**" + - ".github/workflows/changes.yml" + deny: + - '**/Cargo.toml' + - 'Cargo.lock' + - ".github/workflows/deny.yml" + cue: + - 'website/cue/**' + - "vdev/**" + - ".github/workflows/changes.yml" + component_docs: + - 'scripts/generate-component-docs.rb' + - "vdev/**" + - 'website/cue/**/*.cue' + - 'docs/generated/**' + # If changes to the VRL sha is made the combined generated cue file will change which + # may cause issues + - 'Cargo.lock' + - ".github/workflows/changes.yml" + markdown: + - '**/**.md' + - "vdev/**" + - ".github/workflows/changes.yml" + scripts: + - '**/**.sh' + - ".github/workflows/changes.yml" + internal_events: + - 'src/internal_events/**' + - "vdev/**" + - ".github/workflows/changes.yml" + docker: + - 'distribution/docker/**' + - "vdev/**" + - ".github/workflows/changes.yml" + install: + - ".github/workflows/install-sh.yml" + - "distribution/install.sh" + - ".github/workflows/changes.yml" + k8s: + - "src/sources/kubernetes_logs/**" + - "lib/k8s-e2e-tests/**" + - "lib/k8s-test-framework/**" + - "scripts/test-e2e-kubernetes.sh" + - "scripts/build-docker.sh" + - "scripts/deploy-chart-test.sh" + - ".github/workflows/k8s_e2e.yml" + - ".github/workflows/changes.yml" + website: + - "website/**" + not_website: + - "!website/**" + prettier: + - "**/*.yml" + - "**/*.yaml" + - "**/*.js" + - "**/*.ts" + - "**/*.tsx" + - "**/*.json" + - ".prettierrc.json" + - ".prettierignore" + - ".github/workflows/changes.yml" + test-yml: + - ".github/workflows/test.yml" + - ".github/workflows/changes.yml" + integration-yml: + - ".github/workflows/integration.yml" + - ".github/workflows/changes.yml" + test-make-command-yml: + - ".github/workflows/test-make-command.yml" + - ".github/workflows/changes.yml" + msrv-yml: + - ".github/workflows/msrv.yml" + - ".github/workflows/changes.yml" + cross-yml: + - ".github/workflows/cross.yml" + - ".github/workflows/changes.yml" + unit_mac-yml: + - ".github/workflows/unit_mac.yml" + - ".github/workflows/changes.yml" + unit_windows-yml: + - ".github/workflows/unit_windows.yml" + - ".github/workflows/changes.yml" # Detects changes that are specific to integration tests int_tests: @@ -352,7 +364,7 @@ jobs: id: filter with: base: ${{ env.BASE_SHA }} - ref: ${{ env.HEAD_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml # This JSON hack was introduced because GitHub Actions does not support dynamic expressions in the @@ -446,7 +458,7 @@ jobs: id: filter with: base: ${{ env.BASE_SHA }} - ref: ${{ env.HEAD_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml # This JSON hack was introduced because GitHub Actions does not support dynamic expressions in the diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 029ea40e05963..67fae9ea864d7 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -30,7 +30,7 @@ name: CI Integration Review Trigger on: pull_request_review: - types: [ submitted ] + types: [submitted] permissions: contents: read @@ -55,7 +55,7 @@ jobs: timeout-minutes: 5 if: startsWith(github.event.review.body, '/ci-run-integration') || startsWith(github.event.review.body, '/ci-run-e2e') || contains(github.event.review.body, '/ci-run-all') permissions: - statuses: write # Required to set commit status to pending + statuses: write # Required to set commit status to pending steps: - name: Generate authentication token id: generate_token @@ -68,7 +68,7 @@ jobs: uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} - team: 'Vector' + team: "Vector" GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: Validate author membership @@ -86,7 +86,7 @@ jobs: needs: prep-pr permissions: contents: read - packages: write # Required to push test runner image to GHCR + packages: write # Required to push test runner image to GHCR uses: ./.github/workflows/build-test-runner.yml with: commit_sha: ${{ github.event.review.commit_id }} @@ -100,17 +100,47 @@ jobs: timeout-minutes: 90 permissions: contents: read - packages: read # Required to pull test runner image from GHCR + packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false matrix: - service: [ - "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", - "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", - "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "webhdfs" - ] + service: + [ + "amqp", + "appsignal", + "axiom", + "aws", + "azure", + "clickhouse", + "databend", + "datadog-agent", + "datadog-logs", + "datadog-metrics", + "datadog-traces", + "dnstap", + "docker-logs", + "doris", + "elasticsearch", + "eventstoredb", + "fluent", + "gcp", + "greptimedb", + "http-client", + "influxdb", + "kafka", + "logstash", + "loki", + "mqtt", + "mongodb", + "nats", + "nginx", + "opentelemetry", + "postgres", + "prometheus", + "pulsar", + "redis", + "webhdfs" + ] steps: - name: Evaluate run condition id: run_condition @@ -160,13 +190,11 @@ jobs: timeout-minutes: 30 permissions: contents: read - packages: read # Required to pull test runner image from GHCR + packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false matrix: - service: [ - "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" - ] + service: ["datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics"] steps: - name: Evaluate run condition id: run_condition @@ -208,7 +236,6 @@ jobs: bash scripts/environment/prepare.sh --modules=datadog-ci bash scripts/run-integration-test.sh e2e ${{ matrix.service }} - update-pr-status: name: Signal result to PR runs-on: ubuntu-24.04 @@ -218,7 +245,7 @@ jobs: - e2e-tests if: always() && (startsWith(github.event.review.body, '/ci-run-integration') || contains(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-e2e')) permissions: - statuses: write # Required to set final commit status + statuses: write # Required to set final commit status env: FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: @@ -234,7 +261,7 @@ jobs: uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} - team: 'Vector' + team: "Vector" GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: (PR review) Submit PR result as success @@ -243,7 +270,7 @@ jobs: with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} - status: 'success' + status: "success" - name: Check all jobs status run: | diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index ce4e85f392b35..8a2eb206efdcd 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -54,19 +54,19 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 if: | - startsWith(github.event.review.body, '/ci-run-all') - || startsWith(github.event.review.body, '/ci-run-cli') - || startsWith(github.event.review.body, '/ci-run-test-vector-api') - || startsWith(github.event.review.body, '/ci-run-test-behavior') - || startsWith(github.event.review.body, '/ci-run-check-examples') - || startsWith(github.event.review.body, '/ci-run-test-docs') - || startsWith(github.event.review.body, '/ci-run-deny') - || startsWith(github.event.review.body, '/ci-run-component-features') - || startsWith(github.event.review.body, '/ci-run-cross') - || startsWith(github.event.review.body, '/ci-run-unit-mac') - || startsWith(github.event.review.body, '/ci-run-unit-windows') - || startsWith(github.event.review.body, '/ci-run-environment') - || startsWith(github.event.review.body, '/ci-run-k8s') + startsWith(github.event.review.body, '/ci-run-all') + || startsWith(github.event.review.body, '/ci-run-cli') + || startsWith(github.event.review.body, '/ci-run-test-vector-api') + || startsWith(github.event.review.body, '/ci-run-test-behavior') + || startsWith(github.event.review.body, '/ci-run-check-examples') + || startsWith(github.event.review.body, '/ci-run-test-docs') + || startsWith(github.event.review.body, '/ci-run-deny') + || startsWith(github.event.review.body, '/ci-run-component-features') + || startsWith(github.event.review.body, '/ci-run-cross') + || startsWith(github.event.review.body, '/ci-run-unit-mac') + || startsWith(github.event.review.body, '/ci-run-unit-windows') + || startsWith(github.event.review.body, '/ci-run-environment') + || startsWith(github.event.review.body, '/ci-run-k8s') steps: - name: Generate authentication token id: generate_token @@ -79,7 +79,7 @@ jobs: uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} - team: 'Vector' + team: "Vector" GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: Validate author membership diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index 9770b4ab81383..5803451f1dd6c 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -13,17 +13,17 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string schedule: - - cron: '0 10 * * 1' # 10:00 UTC Monday (6 AM EST NYC) + - cron: "0 10 * * 1" # 10:00 UTC Monday (6 AM EST NYC) permissions: contents: read diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a4a4c39d4f760..db19d27ffd582 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,14 +4,14 @@ on: push: branches: - master - workflow_dispatch: # Allow manual trigger from GitHub UI + workflow_dispatch: # Allow manual trigger from GitHub UI permissions: contents: read env: CI: true - CARGO_INCREMENTAL: '0' # Disable incremental compilation for coverage + CARGO_INCREMENTAL: "0" # Disable incremental compilation for coverage jobs: coverage: diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index dbb93e6432a33..6cd7edf89bdb9 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -23,18 +23,18 @@ on: required: true permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: create_preview_site: runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - contents: read # Required for repository context - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required for repository context + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status steps: # Get the artifacts with the PR number and branch name - name: Download artifact diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index 9519a7d79df94..b554a45d45c09 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string diff --git a/.github/workflows/custom_builds.yml b/.github/workflows/custom_builds.yml index 98a7b3647295f..60e1bc8dc6bed 100644 --- a/.github/workflows/custom_builds.yml +++ b/.github/workflows/custom_builds.yml @@ -1,7 +1,7 @@ name: Custom Builds permissions: - contents: read # Restrictive default + contents: read # Restrictive default on: workflow_dispatch: {} @@ -9,8 +9,8 @@ on: jobs: Custom: permissions: - contents: write # Required to create/update releases and tags - packages: write # Required to push container images to GHCR + contents: write # Required to create/update releases and tags + packages: write # Required to push container images to GHCR uses: ./.github/workflows/publish.yml with: git_ref: ${{ github.ref }} diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 52b40d1e1d535..cef811743e716 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -14,14 +14,14 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string @@ -30,7 +30,7 @@ on: types: [checks_requested] schedule: # Same schedule as nightly.yml - - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday + - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday concurrency: group: ${{ github.workflow }}-${{ github.event.number || github.sha }} diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index cf6313e4ce894..de519e0cbc955 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string @@ -15,12 +15,12 @@ on: type: boolean default: false ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string schedule: - - cron: '0 6 * * 1' # Every Monday at 06:00 UTC + - cron: "0 6 * * 1" # Every Monday at 06:00 UTC env: VERBOSE: true diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index 69777ead18d83..7d371baa27388 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -4,13 +4,13 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 48b4deeb218d3..74b782aa7d336 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -16,7 +16,7 @@ on: # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code env: AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 861913c315cd4..64cfe8c6da004 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: pull_request: merge_group: - types: [ checks_requested ] + types: [checks_requested] concurrency: # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue @@ -58,7 +58,7 @@ jobs: needs.changes.outputs.integration-yml == 'true' || needs.changes.outputs.int-tests-any == 'true' || needs.changes.outputs.e2e-tests-any == 'true'))) - }} + }} uses: ./.github/workflows/build-test-runner.yml with: commit_sha: ${{ github.sha }} @@ -75,13 +75,43 @@ jobs: matrix: # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/23474 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. - service: [ - "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", - "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", - "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "webhdfs" - ] + service: + [ + "amqp", + "appsignal", + "axiom", + "aws", + "azure", + "clickhouse", + "databend", + "datadog-agent", + "datadog-logs", + "datadog-metrics", + "datadog-traces", + "dnstap", + "docker-logs", + "doris", + "elasticsearch", + "eventstoredb", + "fluent", + "gcp", + "greptimedb", + "http-client", + "influxdb", + "kafka", + "logstash", + "loki", + "mqtt", + "mongodb", + "nats", + "nginx", + "opentelemetry", + "postgres", + "prometheus", + "pulsar", + "redis", + "webhdfs" + ] timeout-minutes: 90 steps: - name: Download JSON artifact from changes.yml @@ -135,7 +165,6 @@ jobs: command: | bash scripts/run-integration-test.sh int ${{ matrix.service }} - e2e-tests: runs-on: ubuntu-24.04-8core needs: @@ -145,7 +174,7 @@ jobs: (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} strategy: matrix: - service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ] + service: ["datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics"] timeout-minutes: 90 steps: diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 0aa171f0daacd..afdcc01252e20 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -21,20 +21,20 @@ on: workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string pull_request: merge_group: types: [checks_requested] schedule: - - cron: '0 1 * * 2-6' # 01:00 UTC Tue-Sat + - cron: "0 1 * * 2-6" # 01:00 UTC Tue-Sat concurrency: # In flight runs will be canceled through re-trigger in the merge queue, scheduled run, or if diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e94a4bb384ec6..8bfb4b5fb26c9 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -13,7 +13,7 @@ jobs: contents: read pull-requests: write steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - sync-labels: true + - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + sync-labels: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 848bed90f8299..b37bfdbc20e25 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,18 +1,18 @@ name: Nightly permissions: - contents: read # Restrictive default + contents: read # Restrictive default on: workflow_dispatch: schedule: - - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday + - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday jobs: Nightly: permissions: - contents: write # Required to create/update releases and tags - packages: write # Required to push container images to GHCR + contents: write # Required to create/update releases and tags + packages: write # Required to push container images to GHCR uses: ./.github/workflows/publish.yml with: git_ref: ${{ github.ref }} diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 08ff26836ca20..98c5a6a2df720 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -5,15 +5,15 @@ on: # Restrictive default permissions permissions: - contents: read # Required for repository context + contents: read # Required for repository context jobs: approval_check: runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - contents: read # Required for repository context - actions: write # Required to upload artifacts that trigger the preview build workflow + contents: read # Required for repository context + actions: write # Required to upload artifacts that trigger the preview build workflow # Only run for PRs with 'website' in the branch name if: ${{ contains(github.head_ref, 'website') }} steps: diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 06e8411a81541..bc9cca9baf473 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -6,15 +6,15 @@ permissions: on: pull_request: paths: - - "proto/**" - - "lib/vector-core/proto/**" + - "proto/**" + - "lib/vector-core/proto/**" merge_group: types: [checks_requested] concurrency: - # `github.ref` is unique for MQ runs and PRs - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # `github.ref` is unique for MQ runs and PRs + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: validate-protos: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0594151b2db23..4c66ff585e173 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -59,14 +59,14 @@ jobs: strategy: matrix: include: - - target: x86_64-unknown-linux-musl # .tar.gz - - target: x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm - - target: aarch64-unknown-linux-musl # .tar.gz - - target: aarch64-unknown-linux-gnu # .tar.gz, .deb, .rpm - - target: armv7-unknown-linux-gnueabihf # .tar.gz, .deb, .rpm - - target: armv7-unknown-linux-musleabihf # .tar.gz - - target: arm-unknown-linux-gnueabi # .tar.gz, .deb - - target: arm-unknown-linux-musleabi # .tar.gz + - target: x86_64-unknown-linux-musl # .tar.gz + - target: x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm + - target: aarch64-unknown-linux-musl # .tar.gz + - target: aarch64-unknown-linux-gnu # .tar.gz, .deb, .rpm + - target: armv7-unknown-linux-gnueabihf # .tar.gz, .deb, .rpm + - target: armv7-unknown-linux-musleabihf # .tar.gz + - target: arm-unknown-linux-gnueabi # .tar.gz, .deb + - target: arm-unknown-linux-musleabi # .tar.gz steps: - name: Checkout Vector uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1b93487a54ec2..167dc0186bada 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -32,11 +32,11 @@ on: description: "The SHA to use for comparison (optional). If not provided, it defaults to the current HEAD of the origin/master branch." required: false schedule: - - cron: '0 7 * * 1' # Runs at 7 AM UTC on Mondays + - cron: "0 7 * * 1" # Runs at 7 AM UTC on Mondays # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code env: SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }} @@ -44,7 +44,6 @@ env: SMP_REPLICAS: 100 # default is 10 jobs: - resolve-inputs: runs-on: ubuntu-24.04 outputs: @@ -57,7 +56,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 # need to pull repository history to find merge bases + fetch-depth: 0 # need to pull repository history to find merge bases - name: Set and Validate SHAs id: set_and_validate_shas @@ -195,8 +194,8 @@ jobs: - resolve-inputs # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 @@ -238,8 +237,8 @@ jobs: - resolve-inputs # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 @@ -305,7 +304,7 @@ jobs: - confirm-valid-credentials - build-baseline steps: - - name: 'Download baseline image' + - name: "Download baseline image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: baseline-image @@ -345,7 +344,7 @@ jobs: - confirm-valid-credentials - build-comparison steps: - - name: 'Download comparison image' + - name: "Download comparison image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: comparison-image @@ -386,8 +385,8 @@ jobs: - upload-comparison-image-to-ecr # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -502,8 +501,8 @@ jobs: - resolve-inputs # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9965f4d6193c..04192306a1a77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release Suite permissions: - contents: read # Restrictive default + contents: read # Restrictive default on: push: @@ -12,8 +12,8 @@ on: jobs: Release: permissions: - contents: write # Required to create/update releases and tags - packages: write # Required to push container images to GHCR + contents: write # Required to create/update releases and tags + packages: write # Required to push container images to GHCR uses: ./.github/workflows/publish.yml with: git_ref: ${{ github.ref }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index c43853e5a7d51..7a8fd8c9a7d47 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -10,9 +10,9 @@ on: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - - cron: '0 6 * * 1' # 6 AM UTC every Monday + - cron: "0 6 * * 1" # 6 AM UTC every Monday push: - branches: [ "master" ] + branches: ["master"] # Declare default permissions as read only. permissions: read-all diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 425111667a5e8..e95e1183fc593 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -59,13 +59,13 @@ name: Check Spelling on: pull_request_target: branches: - - "**" + - "**" tags-ignore: - - "**" + - "**" types: - - 'opened' - - 'reopened' - - 'synchronize' + - "opened" + - "reopened" + - "synchronize" permissions: contents: read @@ -88,51 +88,50 @@ jobs: # note: If you use only_check_changed_files, you do not want cancel-in-progress cancel-in-progress: true steps: - - name: check-spelling - id: spelling - uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 - with: - suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} - checkout: true - check_file_names: 1 - spell_check_this: check-spelling/spell-check-this@prerelease - post_comment: 0 - use_sarif: ${{ (!github.event.pull_request || (github.event.pull_request.head.repo.full_name == github.repository)) && 1 }} - extra_dictionary_limit: 20 - extra_dictionaries: - cspell:aws/aws.txt - cspell:cpp/src/compiler-clang-attributes.txt - cspell:cpp/src/compiler-msvc.txt - cspell:cpp/src/ecosystem.txt - cspell:cpp/src/lang-jargon.txt - cspell:cpp/src/stdlib-cpp.txt - cspell:css/dict/css.txt - cspell:django/dict/django.txt - cspell:docker/src/docker-words.txt - cspell:dotnet/dict/dotnet.txt - cspell:elixir/dict/elixir.txt - cspell:filetypes/filetypes.txt - cspell:fullstack/dict/fullstack.txt - cspell:golang/dict/go.txt - cspell:html-symbol-entities/entities.txt - cspell:html/dict/html.txt - cspell:java/src/java-terms.txt - cspell:java/src/java.txt - cspell:k8s/dict/k8s.txt - cspell:lorem-ipsum/dictionary.txt - cspell:lua/dict/lua.txt - cspell:node/dict/node.txt - cspell:npm/dict/npm.txt - cspell:php/dict/php.txt - cspell:public-licenses/src/generated/public-licenses.txt - cspell:python/src/common/extra.txt - cspell:python/src/python/python-lib.txt - cspell:python/src/python/python.txt - cspell:r/src/r.txt - cspell:ruby/dict/ruby.txt - cspell:rust/dict/rust.txt - cspell:shell/dict/shell-all-words.txt - cspell:software-terms/dict/softwareTerms.txt - cspell:swift/src/swift.txt - cspell:typescript/dict/typescript.txt - check_extra_dictionaries: '' + - name: check-spelling + id: spelling + uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 + with: + suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} + checkout: true + check_file_names: 1 + spell_check_this: check-spelling/spell-check-this@prerelease + post_comment: 0 + use_sarif: ${{ (!github.event.pull_request || (github.event.pull_request.head.repo.full_name == github.repository)) && 1 }} + extra_dictionary_limit: 20 + extra_dictionaries: cspell:aws/aws.txt + cspell:cpp/src/compiler-clang-attributes.txt + cspell:cpp/src/compiler-msvc.txt + cspell:cpp/src/ecosystem.txt + cspell:cpp/src/lang-jargon.txt + cspell:cpp/src/stdlib-cpp.txt + cspell:css/dict/css.txt + cspell:django/dict/django.txt + cspell:docker/src/docker-words.txt + cspell:dotnet/dict/dotnet.txt + cspell:elixir/dict/elixir.txt + cspell:filetypes/filetypes.txt + cspell:fullstack/dict/fullstack.txt + cspell:golang/dict/go.txt + cspell:html-symbol-entities/entities.txt + cspell:html/dict/html.txt + cspell:java/src/java-terms.txt + cspell:java/src/java.txt + cspell:k8s/dict/k8s.txt + cspell:lorem-ipsum/dictionary.txt + cspell:lua/dict/lua.txt + cspell:node/dict/node.txt + cspell:npm/dict/npm.txt + cspell:php/dict/php.txt + cspell:public-licenses/src/generated/public-licenses.txt + cspell:python/src/common/extra.txt + cspell:python/src/python/python-lib.txt + cspell:python/src/python/python.txt + cspell:r/src/r.txt + cspell:ruby/dict/ruby.txt + cspell:rust/dict/rust.txt + cspell:shell/dict/shell-all-words.txt + cspell:software-terms/dict/softwareTerms.txt + cspell:swift/src/swift.txt + cspell:typescript/dict/typescript.txt + check_extra_dictionaries: "" diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index d3330f07b7280..04ed6b19d9f25 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,8 +4,8 @@ on: push: branches: - master - - "**" # Run on all branches (includes PR branches) - workflow_dispatch: # Allow manual trigger from GitHub UI + - "**" # Run on all branches (includes PR branches) + workflow_dispatch: # Allow manual trigger from GitHub UI permissions: contents: read diff --git a/.github/workflows/test-make-command.yml b/.github/workflows/test-make-command.yml index f10ceaa3c45ad..79c27d54792f4 100644 --- a/.github/workflows/test-make-command.yml +++ b/.github/workflows/test-make-command.yml @@ -4,29 +4,28 @@ on: workflow_call: inputs: command: - description: 'Make command to run (e.g., test-behavior, check-examples, test-docs)' + description: "Make command to run (e.g., test-behavior, check-examples, test-docs)" required: true type: string job_name: - description: 'Display name for the job/status check' + description: "Display name for the job/status check" required: true type: string ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string cargo_nextest: - description: 'Whether to install cargo-nextest' + description: "Whether to install cargo-nextest" required: false type: boolean default: false upload_test_results: - description: 'Whether to upload nextest test results (for commands that use cargo nextest)' + description: "Whether to upload nextest test results (for commands that use cargo nextest)" required: false type: boolean default: false - permissions: contents: read diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 986b145305f78..81ceef93ca5f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ on: # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code concurrency: # `github.ref` is unique for MQ runs and PRs @@ -31,13 +31,14 @@ jobs: check-fmt: name: Check code format runs-on: ubuntu-24.04 - if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} + if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' || needs.changes.outputs.prettier == 'true' }} needs: changes steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true + prettier: true - run: make check-fmt check-clippy: diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index b177b018d13ef..e0d367b530959 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -4,11 +4,10 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string - permissions: contents: read diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml index 831eec042f869..a4553b4a0b751 100644 --- a/.github/workflows/unit_windows.yml +++ b/.github/workflows/unit_windows.yml @@ -4,11 +4,10 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string - permissions: contents: read diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml index edb2f36674731..3a46822098beb 100644 --- a/.github/workflows/vdev_publish.yml +++ b/.github/workflows/vdev_publish.yml @@ -1,16 +1,16 @@ name: Publish vdev on: push: - tags: [ "vdev-v*.*.*" ] + tags: ["vdev-v*.*.*"] permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: build: runs-on: ${{ matrix.os }} permissions: - contents: write # Required to upload release assets + contents: write # Required to upload release assets strategy: matrix: include: diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000000..f7b79d1ba7d52 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,30 @@ +# Build artifacts +target/ +node_modules/ + +# Generated / test fixture files +docs/generated/ +lib/codecs/tests/data/ + +# Hugo templates and assets (Go template syntax breaks prettier) +website/**/*.html +website/assets/js/app.js +website/assets/js/dd-browser-logs-rum.js +website/layouts/ + +# Hugo build output and generated resources +website/public/ +website/resources/ +website/data/ +website/assets/jsconfig.json + +# CUE files (not supported by prettier) +website/cue/ + +# Config examples using VRL triple-quoted strings +config/examples/docs_example.yaml +config/examples/file_to_prometheus.yaml + +# Lock files +Cargo.lock +website/yarn.lock diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000000000..a4643ee91d31c --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,4 @@ +{ + "trailingComma": "none", + "printWidth": 120 +} diff --git a/Makefile b/Makefile index 5c52c367c11b7..a923806254d9f 100644 --- a/Makefile +++ b/Makefile @@ -510,6 +510,24 @@ check-markdown: ## Check that markdown is styled properly fix-markdown: ## Auto-fix markdown style issues ${MAYBE_ENVIRONMENT_EXEC} markdownlint-cli2 --fix $(shell git ls-files '*.md') +.PHONY: check-prettier +check-prettier: ## Check that JS/TS/YAML/JSON files are formatted with prettier + @for ext in yml yaml js ts tsx json; do \ + files=$$(git ls-files "*.$$ext"); \ + if [ -n "$$files" ]; then \ + ${MAYBE_ENVIRONMENT_EXEC} prettier --ignore-path .prettierignore --check $$files || exit 1; \ + fi; \ + done + +.PHONY: fix-prettier +fix-prettier: ## Auto-fix JS/TS/YAML/JSON formatting with prettier + @for ext in yml yaml js ts tsx json; do \ + files=$$(git ls-files "*.$$ext"); \ + if [ -n "$$files" ]; then \ + ${MAYBE_ENVIRONMENT_EXEC} prettier --ignore-path .prettierignore --write $$files || exit 1; \ + fi; \ + done + .PHONY: check-examples check-examples: ## Check that the config/examples files are valid ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check examples diff --git a/config/examples/environment_variables.yaml b/config/examples/environment_variables.yaml index 2f73528397ad1..d8421c1cc8e01 100644 --- a/config/examples/environment_variables.yaml +++ b/config/examples/environment_variables.yaml @@ -13,7 +13,7 @@ data_dir: "/var/lib/vector" sources: apache_logs: type: "file" - include: [ "/var/log/apache2/*.log" ] + include: ["/var/log/apache2/*.log"] # ignore files older than 1 day ignore_older_secs: 86400 @@ -21,7 +21,7 @@ sources: # Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: add_host: - inputs: [ "apache_logs" ] + inputs: ["apache_logs"] type: "remap" source: | ''' @@ -32,7 +32,7 @@ transforms: # Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: - inputs: [ "add_host" ] + inputs: ["add_host"] type: "console" encoding: codec: "json" diff --git a/config/examples/file_to_cloudwatch_metrics.yaml b/config/examples/file_to_cloudwatch_metrics.yaml index f322ab8f091b0..138f9d20ceac2 100644 --- a/config/examples/file_to_cloudwatch_metrics.yaml +++ b/config/examples/file_to_cloudwatch_metrics.yaml @@ -8,13 +8,13 @@ data_dir: "/var/lib/vector" sources: file: type: "file" - include: [ "sample.log" ] + include: ["sample.log"] start_at_beginning: true # Structure and parse the data transforms: remap: - inputs: [ "file" ] + inputs: ["file"] type: "remap" drop_on_error: false source: | @@ -22,7 +22,7 @@ transforms: # Transform into metrics log_to_metric: - inputs: [ "remap" ] + inputs: ["remap"] type: "log_to_metric" metrics: - type: "counter" @@ -35,19 +35,19 @@ transforms: # Output data sinks: console_metrics: - inputs: [ "log_to_metric" ] + inputs: ["log_to_metric"] type: "console" encoding: codec: "json" console_logs: - inputs: [ "remap" ] + inputs: ["remap"] type: "console" encoding: codec: "json" cloudwatch: - inputs: [ "log_to_metric" ] + inputs: ["log_to_metric"] type: "aws_cloudwatch_metrics" namespace: "vector" endpoint: "http://localhost:4566" diff --git a/config/examples/namespacing/sinks/es_cluster.yaml b/config/examples/namespacing/sinks/es_cluster.yaml index 820a49080b474..f916dac1b7180 100644 --- a/config/examples/namespacing/sinks/es_cluster.yaml +++ b/config/examples/namespacing/sinks/es_cluster.yaml @@ -1,6 +1,6 @@ # Send structured data to a short-term storage -inputs: ["apache_sample"] # only take sampled data +inputs: ["apache_sample"] # only take sampled data type: "elasticsearch" -endpoint: "http://79.12.221.222:9200" # local or external host +endpoint: "http://79.12.221.222:9200" # local or external host bulk: - index: "vector-%Y-%m-%d" # daily indices + index: "vector-%Y-%m-%d" # daily indices diff --git a/config/examples/namespacing/sinks/s3_archives.yaml b/config/examples/namespacing/sinks/s3_archives.yaml index a72c27027e8a1..6de3cd4df8f56 100644 --- a/config/examples/namespacing/sinks/s3_archives.yaml +++ b/config/examples/namespacing/sinks/s3_archives.yaml @@ -1,13 +1,13 @@ # Send structured data to a cost-effective long-term storage -inputs: ["apache_parser"] # don't sample for S3 +inputs: ["apache_parser"] # don't sample for S3 type: "aws_s3" region: "us-east-1" bucket: "my-log-archives" -key_prefix: "date=%Y-%m-%d" # daily partitions, hive friendly format -compression: "gzip" # compress final objects +key_prefix: "date=%Y-%m-%d" # daily partitions, hive friendly format +compression: "gzip" # compress final objects framing: - method: "newline_delimited" # new line delimited... + method: "newline_delimited" # new line delimited... encoding: - codec: "json" # ...JSON + codec: "json" # ...JSON batch: - max_bytes: 10000000 # 10mb uncompressed + max_bytes: 10000000 # 10mb uncompressed diff --git a/config/examples/namespacing/transforms/apache_sample.yaml b/config/examples/namespacing/transforms/apache_sample.yaml index 91b789e1016f2..cca25cf0989af 100644 --- a/config/examples/namespacing/transforms/apache_sample.yaml +++ b/config/examples/namespacing/transforms/apache_sample.yaml @@ -1,4 +1,4 @@ # Sample the data to save on cost inputs: ["apache_parser"] type: "sample" -rate: 2 # only keep 50% (1/`rate`) +rate: 2 # only keep 50% (1/`rate`) diff --git a/config/examples/stdio.yaml b/config/examples/stdio.yaml index cbe6429d8afae..7ca39a81876da 100644 --- a/config/examples/stdio.yaml +++ b/config/examples/stdio.yaml @@ -11,7 +11,7 @@ sources: sinks: out: - inputs: [ "in" ] + inputs: ["in"] type: "console" encoding: codec: "text" diff --git a/config/examples/wrapped_json.yaml b/config/examples/wrapped_json.yaml index 90fed711f2a18..4bd6ab0a79f58 100644 --- a/config/examples/wrapped_json.yaml +++ b/config/examples/wrapped_json.yaml @@ -12,14 +12,14 @@ data_dir: "/var/lib/vector" sources: logs: type: "file" - include: [ "/var/log/*.log" ] + include: ["/var/log/*.log"] ignore_older_secs: 86400 # 1 day # Parse the data as JSON # Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: parse_json: - inputs: [ "logs" ] + inputs: ["logs"] type: "remap" drop_on_error: false source: | @@ -36,7 +36,7 @@ transforms: # Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: - inputs: [ "parse_json" ] + inputs: ["parse_json"] type: "console" encoding: codec: "json" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml index 4d913387252ef..4dd744f5d2c02 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml @@ -3,8 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.logs.v1.LogsService.Export - post: /v1/logs - body: "*" - + rules: + - selector: opentelemetry.proto.collector.logs.v1.LogsService.Export + post: /v1/logs + body: "*" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml index 7ff39d4b406c6..7643be4125831 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml @@ -3,8 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.metrics.v1.MetricsService.Export - post: /v1/metrics - body: "*" - + rules: + - selector: opentelemetry.proto.collector.metrics.v1.MetricsService.Export + post: /v1/metrics + body: "*" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml index d091b3a8d53d7..7eaa4e548c33a 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml @@ -3,7 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export - post: /v1/traces - body: "*" + rules: + - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export + post: /v1/traces + body: "*" diff --git a/lib/vector-core/tests/data/fixtures/log_event/malformed.json b/lib/vector-core/tests/data/fixtures/log_event/malformed.json index 6f956957a7b81..32603adb1fb2f 100644 --- a/lib/vector-core/tests/data/fixtures/log_event/malformed.json +++ b/lib/vector-core/tests/data/fixtures/log_event/malformed.json @@ -1,4 +1,4 @@ { "foo": "bar", "foo": "You're not allowed duplicate keys..." -} \ No newline at end of file +} diff --git a/lib/vector-vrl/tests/resources/json-schema_definition.json b/lib/vector-vrl/tests/resources/json-schema_definition.json index 9c4c8a00d0b28..a2e51e99c51d1 100644 --- a/lib/vector-vrl/tests/resources/json-schema_definition.json +++ b/lib/vector-vrl/tests/resources/json-schema_definition.json @@ -1 +1,10 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productUser":{"description":"The unique identifier for a product user","type":"string","format":"email"}}} +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Product", + "description": "A product from Acme's catalog", + "type": "object", + "properties": { + "productUser": { "description": "The unique identifier for a product user", "type": "string", "format": "email" } + } +} diff --git a/lib/vector-vrl/web-playground/public/index.js b/lib/vector-vrl/web-playground/public/index.js index 4cdc2bf4e0dcc..9f810ec29e74a 100644 --- a/lib/vector-vrl/web-playground/public/index.js +++ b/lib/vector-vrl/web-playground/public/index.js @@ -1,5 +1,5 @@ -import init, {run_vrl, vector_link, vector_version, vrl_link, vrl_version} from "./pkg/vector_vrl_web_playground.js"; -import {vrlLanguageDefinition, vrlThemeDefinition} from "./vrl-highlighter.js"; +import init, { run_vrl, vector_link, vector_version, vrl_link, vrl_version } from "./pkg/vector_vrl_web_playground.js"; +import { vrlLanguageDefinition, vrlThemeDefinition } from "./vrl-highlighter.js"; const PROGRAM_EDITOR_DEFAULT_VALUE = `# Remove some fields del(.foo) @@ -42,309 +42,313 @@ You can try validating your JSON here: https://jsonlint.com/ `; function loadMonaco() { - return new Promise((resolve, reject) => { - // require is provided by loader.min.js. - require.config({ - paths: {vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs"}, - }); - require(["vs/editor/editor.main"], () => resolve(window.monaco), reject); + return new Promise((resolve, reject) => { + // require is provided by loader.min.js. + require.config({ + paths: { vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs" } }); + require(["vs/editor/editor.main"], () => resolve(window.monaco), reject); + }); } export class VrlWebPlayground { - static async create() { - const instance = new VrlWebPlayground(true); - await instance._initAsync(); - return instance; + static async create() { + const instance = new VrlWebPlayground(true); + await instance._initAsync(); + return instance; + } + + constructor(_internal = false) { + if (!_internal) { + // Prefer factory: VrlWebPlayground.create() + this._initAsync(); // fire-and-forget fallback } - - constructor(_internal = false) { - if (!_internal) { - // Prefer factory: VrlWebPlayground.create() - this._initAsync(); // fire-and-forget fallback - } + } + + async _initAsync() { + // Load wasm/runtime + await init(); + + // Bind native funcs/versions + this.run_vrl = run_vrl; + this.vector_version = vector_version(); + this.vector_link = vector_link(); + this.vrl_version = vrl_version(); + this.vrl_link = vrl_link(); + + // Load Monaco + this.monaco = await loadMonaco(); + + // VRL lang + theme + this.monaco.languages.register({ id: "vrl" }); + this.monaco.editor.defineTheme("vrl-theme", vrlThemeDefinition); + this.monaco.languages.setMonarchTokensProvider("vrl", vrlLanguageDefinition); + + // Editors + this.eventEditor = this.createDefaultEditor("container-event", EVENT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); + this.outputEditor = this.createDefaultEditor("container-output", OUTPUT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); + this.programEditor = this.createDefaultEditor( + "container-program", + PROGRAM_EDITOR_DEFAULT_VALUE, + "vrl", + "vrl-theme" + ); + + // Versions + this.addVersions(); + + // Handle shared state from URL (if present) + this._maybeLoadFromUrl(); + } + + _maybeLoadFromUrl() { + const qs = window.location.search; + if (!qs) return; + + const urlParams = new URLSearchParams(qs); + const stateParam = urlParams.get("state"); + if (!stateParam) return; + + try { + const decoded = atob(decodeURIComponent(stateParam)); + const urlState = JSON.parse(decoded); + + if (typeof urlState.program === "string") { + this.programEditor.setValue(urlState.program); + } + + if (urlState.is_jsonl === true && typeof urlState.event === "string") { + this.eventEditor.setValue(urlState.event); + } else if (urlState.event != null) { + this.eventEditor.setValue(JSON.stringify(urlState.event, null, "\t")); + } + + // Run immediately with the provided state + this.handleRunCode(urlState); + } catch (e) { + this.disableJsonLinting(); + this.outputEditor.setValue(`Error reading the shared URL\n${e}`); } + } - async _initAsync() { - // Load wasm/runtime - await init(); - - // Bind native funcs/versions - this.run_vrl = run_vrl; - this.vector_version = vector_version(); - this.vector_link = vector_link(); - this.vrl_version = vrl_version(); - this.vrl_link = vrl_link(); - - // Load Monaco - this.monaco = await loadMonaco(); - - // VRL lang + theme - this.monaco.languages.register({id: "vrl"}); - this.monaco.editor.defineTheme("vrl-theme", vrlThemeDefinition); - this.monaco.languages.setMonarchTokensProvider("vrl", vrlLanguageDefinition); - - // Editors - this.eventEditor = this.createDefaultEditor("container-event", EVENT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); - this.outputEditor = this.createDefaultEditor("container-output", OUTPUT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); - this.programEditor = this.createDefaultEditor("container-program", PROGRAM_EDITOR_DEFAULT_VALUE, "vrl", "vrl-theme"); - - // Versions - this.addVersions(); - - // Handle shared state from URL (if present) - this._maybeLoadFromUrl(); + addVersions() { + const vectorLinkElement = document.getElementById("vector-version-link"); + if (vectorLinkElement) { + vectorLinkElement.text = (this.vector_version || "").toString().substring(0, 8); + vectorLinkElement.href = this.vector_link || "#"; } - _maybeLoadFromUrl() { - const qs = window.location.search; - if (!qs) return; - - const urlParams = new URLSearchParams(qs); - const stateParam = urlParams.get("state"); - if (!stateParam) return; - - try { - const decoded = atob(decodeURIComponent(stateParam)); - const urlState = JSON.parse(decoded); - - if (typeof urlState.program === "string") { - this.programEditor.setValue(urlState.program); - } - - if (urlState.is_jsonl === true && typeof urlState.event === "string") { - this.eventEditor.setValue(urlState.event); - } else if (urlState.event != null) { - this.eventEditor.setValue(JSON.stringify(urlState.event, null, "\t")); - } - - // Run immediately with the provided state - this.handleRunCode(urlState); - } catch (e) { - this.disableJsonLinting(); - this.outputEditor.setValue(`Error reading the shared URL\n${e}`); - } + const vrlLinkElement = document.getElementById("vrl-version-link"); + if (vrlLinkElement) { + vrlLinkElement.text = (this.vrl_version || "").toString().substring(0, 8); + vrlLinkElement.href = this.vrl_link || "#"; } + } - addVersions() { - const vectorLinkElement = document.getElementById("vector-version-link"); - if (vectorLinkElement) { - vectorLinkElement.text = (this.vector_version || "").toString().substring(0, 8); - vectorLinkElement.href = this.vector_link || "#"; - } - - const vrlLinkElement = document.getElementById("vrl-version-link"); - if (vrlLinkElement) { - vrlLinkElement.text = (this.vrl_version || "").toString().substring(0, 8); - vrlLinkElement.href = this.vrl_link || "#"; - } + createDefaultEditor(elementId, value, language, theme) { + const el = document.getElementById(elementId); + if (!el) { + console.warn(`Editor container #${elementId} not found`); + return null; } + return this.monaco.editor.create(el, { + value, + language, + theme, + minimap: { enabled: false }, + automaticLayout: true, + wordWrap: "on" + }); + } - createDefaultEditor(elementId, value, language, theme) { - const el = document.getElementById(elementId); - if (!el) { - console.warn(`Editor container #${elementId} not found`); - return null; - } - return this.monaco.editor.create(el, { - value, - language, - theme, - minimap: {enabled: false}, - automaticLayout: true, - wordWrap: 'on', - }); + _clearOutput() { + if (this.outputEditor) { + // wipe the buffer so stale values never linger + this.outputEditor.setValue(""); } - - _clearOutput() { - if (this.outputEditor) { - // wipe the buffer so stale values never linger - this.outputEditor.setValue(""); - } - const elapsedEl = document.getElementById("elapsed-time"); - if (elapsedEl) { - elapsedEl.textContent = ""; - } + const elapsedEl = document.getElementById("elapsed-time"); + if (elapsedEl) { + elapsedEl.textContent = ""; } + } - _formatRunResult(runResult) { - if (runResult?.target_value != null) { - const isJson = typeof runResult.target_value === "object"; - const text = isJson - ? JSON.stringify(runResult.target_value, null, "\t") - : String(runResult.target_value); - return {text, isJson}; - } - if (runResult?.msg != null) { - return {text: String(runResult.msg), isJson: false}; - } - return {text: "Error - VRL did not return a result.", isJson: false}; + _formatRunResult(runResult) { + if (runResult?.target_value != null) { + const isJson = typeof runResult.target_value === "object"; + const text = isJson ? JSON.stringify(runResult.target_value, null, "\t") : String(runResult.target_value); + return { text, isJson }; } - - _setElapsed(elapsed_time) { - const elapsedEl = document.getElementById("elapsed-time"); - if (elapsedEl && elapsed_time != null) { - const ms = elapsed_time.toFixed(4) - elapsedEl.textContent = `Duration: ${ms} milliseconds`; - } + if (runResult?.msg != null) { + return { text: String(runResult.msg), isJson: false }; } - - _safeGet(editor, fallback = "") { - return editor?.getValue?.() ?? fallback; + return { text: "Error - VRL did not return a result.", isJson: false }; + } + + _setElapsed(elapsed_time) { + const elapsedEl = document.getElementById("elapsed-time"); + if (elapsedEl && elapsed_time != null) { + const ms = elapsed_time.toFixed(4); + elapsedEl.textContent = `Duration: ${ms} milliseconds`; } - - getState() { - if (this.eventEditorIsJsonl()) { - return { - program: this._safeGet(this.programEditor), - event: this.eventEditor.getModel().getLinesContent().join("\n"), - is_jsonl: true, - error: null, - }; - } - - const editorValue = this._safeGet(this.eventEditor); - try { - return { - program: this._safeGet(this.programEditor), - event: JSON.parse(editorValue.length === 0 ? "{}" : editorValue), - is_jsonl: false, - error: null, - }; - } catch (_err) { - return { - program: this._safeGet(this.programEditor), - event: null, - is_jsonl: false, - error: `Could not parse JSON event:\n${editorValue}`, - }; - } + } + + _safeGet(editor, fallback = "") { + return editor?.getValue?.() ?? fallback; + } + + getState() { + if (this.eventEditorIsJsonl()) { + return { + program: this._safeGet(this.programEditor), + event: this.eventEditor.getModel().getLinesContent().join("\n"), + is_jsonl: true, + error: null + }; } - disableJsonLinting() { - this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({validate: false}); + const editorValue = this._safeGet(this.eventEditor); + try { + return { + program: this._safeGet(this.programEditor), + event: JSON.parse(editorValue.length === 0 ? "{}" : editorValue), + is_jsonl: false, + error: null + }; + } catch (_err) { + return { + program: this._safeGet(this.programEditor), + event: null, + is_jsonl: false, + error: `Could not parse JSON event:\n${editorValue}` + }; } - - enableJsonLinting() { - this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({validate: true}); + } + + disableJsonLinting() { + this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: false }); + } + + enableJsonLinting() { + this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: true }); + } + + tryJsonParse(str) { + try { + return JSON.parse(str); + } catch (_e) { + this.disableJsonLinting(); + const err = ERROR_INVALID_JSONL_EVENT_MSG.toString().replace("{{str}}", str); + this.outputEditor.setValue(err); + throw new Error(err); } - - tryJsonParse(str) { - try { - return JSON.parse(str); - } catch (_e) { - this.disableJsonLinting(); - const err = ERROR_INVALID_JSONL_EVENT_MSG.toString().replace("{{str}}", str); - this.outputEditor.setValue(err); - throw new Error(err); - } + } + + /** + * Treat as JSONL if there are >1 non-empty lines and at least the second non-empty + * line *appears* to be a JSON object. Robust to whitespace. + */ + eventEditorIsJsonl() { + const model = this.eventEditor?.getModel?.(); + if (!model) return false; + + const rawLines = model.getLinesContent(); + const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); + if (lines.length <= 1) return false; + + const second = lines[1]; + return second.startsWith("{") && second.endsWith("}"); + } + + _getTimezoneOrDefault() { + const tzEl = document.getElementById("timezone-input"); + return tzEl?.value && tzEl.value.trim().length > 0 ? tzEl.value.trim() : "Default"; + } + + handleRunCode(input) { + this._clearOutput(); + + // JSONL path short-circuit + if (this.eventEditorIsJsonl()) { + return this.handleRunCodeJsonl(); } - /** - * Treat as JSONL if there are >1 non-empty lines and at least the second non-empty - * line *appears* to be a JSON object. Robust to whitespace. - */ - eventEditorIsJsonl() { - const model = this.eventEditor?.getModel?.(); - if (!model) return false; - - const rawLines = model.getLinesContent(); - const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); - if (lines.length <= 1) return false; - - const second = lines[1]; - return second.startsWith("{") && second.endsWith("}"); + if (input == null) { + input = this.getState(); } - _getTimezoneOrDefault() { - const tzEl = document.getElementById("timezone-input"); - return tzEl?.value && tzEl.value.trim().length > 0 ? tzEl.value.trim() : "Default"; + if (input.error) { + console.error(input.error); + this.disableJsonLinting(); + this.outputEditor.setValue(input.error); + return input; } - handleRunCode(input) { - this._clearOutput(); - - // JSONL path short-circuit - if (this.eventEditorIsJsonl()) { - return this.handleRunCodeJsonl(); - } - - if (input == null) { - input = this.getState(); - } - - if (input.error) { - console.error(input.error); - this.disableJsonLinting(); - this.outputEditor.setValue(input.error); - return input; - } - - const timezone = this._getTimezoneOrDefault(); - console.debug("Selected timezone: ", timezone); - const runResult = this.run_vrl(input, timezone); - console.log("Run result: ", runResult); - - const {text, isJson} = this._formatRunResult(runResult); - if (isJson) this.enableJsonLinting(); else this.disableJsonLinting(); - this.outputEditor.setValue(text); - - this._setElapsed(runResult?.elapsed_time); - return runResult; - } - - handleRunCodeJsonl() { - this._clearOutput(); - - const program = this._safeGet(this.programEditor); - const model = this.eventEditor?.getModel?.(); - const rawLines = model ? model.getLinesContent() : []; - const lines = rawLines.map(l => l.trim()).filter(l => l.length > 0); - - const timezone = this._getTimezoneOrDefault(); - - // Build inputs while validating JSON per line - const inputs = lines.map(line => ({ - program, - event: this.tryJsonParse(line), - is_jsonl: true, - })); - - // Run and collect results - const results = inputs.map(input => this.run_vrl(input, timezone)); - - const outputs = results.map(r => this._formatRunResult(r).text); - - // Output is not pure JSON (multiple objects / possible errors) - this.disableJsonLinting(); - this.outputEditor.setValue(outputs.join("\n")); - - // Aggregate elapsed time, rounded - const total = results.reduce((sum, r) => sum + (typeof r?.elapsed_time === "number" ? r.elapsed_time : 0), 0); - this._setElapsed(total); - - return results; - } - - handleShareCode() { - const state = this.getState(); - try { - const encoded = encodeURIComponent(btoa(JSON.stringify(state))); - window.history.pushState(state, "", `?state=${encoded}`); - return true; - } catch (e) { - this.disableJsonLinting(); - this.outputEditor.setValue(`Error encoding state for URL\n${e}`); - return false; - } + const timezone = this._getTimezoneOrDefault(); + console.debug("Selected timezone: ", timezone); + const runResult = this.run_vrl(input, timezone); + console.log("Run result: ", runResult); + + const { text, isJson } = this._formatRunResult(runResult); + if (isJson) this.enableJsonLinting(); + else this.disableJsonLinting(); + this.outputEditor.setValue(text); + + this._setElapsed(runResult?.elapsed_time); + return runResult; + } + + handleRunCodeJsonl() { + this._clearOutput(); + + const program = this._safeGet(this.programEditor); + const model = this.eventEditor?.getModel?.(); + const rawLines = model ? model.getLinesContent() : []; + const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); + + const timezone = this._getTimezoneOrDefault(); + + // Build inputs while validating JSON per line + const inputs = lines.map((line) => ({ + program, + event: this.tryJsonParse(line), + is_jsonl: true + })); + + // Run and collect results + const results = inputs.map((input) => this.run_vrl(input, timezone)); + + const outputs = results.map((r) => this._formatRunResult(r).text); + + // Output is not pure JSON (multiple objects / possible errors) + this.disableJsonLinting(); + this.outputEditor.setValue(outputs.join("\n")); + + // Aggregate elapsed time, rounded + const total = results.reduce((sum, r) => sum + (typeof r?.elapsed_time === "number" ? r.elapsed_time : 0), 0); + this._setElapsed(total); + + return results; + } + + handleShareCode() { + const state = this.getState(); + try { + const encoded = encodeURIComponent(btoa(JSON.stringify(state))); + window.history.pushState(state, "", `?state=${encoded}`); + return true; + } catch (e) { + this.disableJsonLinting(); + this.outputEditor.setValue(`Error encoding state for URL\n${e}`); + return false; } + } } // Prefer the async factory to ensure everything is loaded before use: VrlWebPlayground.create() - .then((instance) => { - window.vrlPlayground = instance; - }) - .catch((err) => { - console.error("Failed to initialize VrlWebPlayground:", err); - }); + .then((instance) => { + window.vrlPlayground = instance; + }) + .catch((err) => { + console.error("Failed to initialize VrlWebPlayground:", err); + }); diff --git a/lib/vector-vrl/web-playground/public/vrl-highlighter.js b/lib/vector-vrl/web-playground/public/vrl-highlighter.js index 3efaed67a4a4b..527934106530f 100644 --- a/lib/vector-vrl/web-playground/public/vrl-highlighter.js +++ b/lib/vector-vrl/web-playground/public/vrl-highlighter.js @@ -1,636 +1,633 @@ // VRL Language Definition export let vrlLanguageDefinition = { - defaultToken: "invalid", - ignoreCase: true, - tokenPostfix: ".vrl", + defaultToken: "invalid", + ignoreCase: true, + tokenPostfix: ".vrl", - brackets: [ - { open: "{", close: "}", token: "delimiter.curly" }, - { open: "[", close: "]", token: "delimiter.square" }, - { open: "(", close: ")", token: "delimiter.parenthesis" }, - ], + brackets: [ + { open: "{", close: "}", token: "delimiter.curly" }, + { open: "[", close: "]", token: "delimiter.square" }, + { open: "(", close: ")", token: "delimiter.parenthesis" } + ], - regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/, + regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/, - keywords: [ - "abort", - "as", - "break", - "continue", - "else", - "false", - "for", - "if", - "impl", - "in", - "let", - "loop", - "null", - "return", - "self", - "std", - "then", - "this", - "true", - "type", - "until", - "use", - "while", - ], + keywords: [ + "abort", + "as", + "break", + "continue", + "else", + "false", + "for", + "if", + "impl", + "in", + "let", + "loop", + "null", + "return", + "self", + "std", + "then", + "this", + "true", + "type", + "until", + "use", + "while" + ], - // we include these common regular expressions - symbols: /[=>[a-z]+)\\.(?P[a-z]+)')" log2metric: type: "log_to_metric" - inputs: [ "remap" ] + inputs: ["remap"] metrics: - - type: "counter" + - type: "counter" field: "procid" tags: hostname: "{{ hostname }}" @@ -41,13 +41,13 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" datadog_metrics: - type: "datadog_metrics" - inputs: [ "log2metric" ] - endpoint: "http://0.0.0.0:8080" - default_api_key: "DEADBEEF" + type: "datadog_metrics" + inputs: ["log2metric"] + endpoint: "http://0.0.0.0:8080" + default_api_key: "DEADBEEF" default_namespace: "vector" diff --git a/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml b/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml +++ b/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml b/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml index e4d63f4e23b41..8c5874e22b718 100644 --- a/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml +++ b/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Sinks @@ -20,16 +20,16 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" splunk_hec_logs: - type: "splunk_hec_logs" - inputs: [ "syslog" ] + type: "splunk_hec_logs" + inputs: ["syslog"] endpoint: "http://0.0.0.0:8080" encoding: codec: "json" - token: "abcd1234" + token: "abcd1234" healthcheck: enabled: false diff --git a/regression/config.yaml b/regression/config.yaml index e7a3f5b1e5dab..9b80bae63a965 100644 --- a/regression/config.yaml +++ b/regression/config.yaml @@ -3,7 +3,6 @@ lading: target: - # Link templates for reports. # # Values may be removed to disable corresponding links in reports. diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 397fefbb464a8..e74aba915ad5f 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -38,6 +38,7 @@ DD_RUST_LICENSE_TOOL_VERSION="1.0.6" CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" MARKDOWNLINT_CLI2_VERSION="0.22.0" +PRETTIER_VERSION="3.8.1" DATADOG_CI_VERSION="5.9.0" VDEV_VERSION="0.3.0" @@ -53,6 +54,7 @@ ALL_MODULES=( dd-rust-license-tool wasm-pack markdownlint-cli2 + prettier datadog-ci release-flags # Not a tool - sources release-flags.sh to set CI env vars vdev @@ -93,6 +95,7 @@ Modules: dd-rust-license-tool wasm-pack markdownlint-cli2 + prettier datadog-ci vdev @@ -219,4 +222,5 @@ maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" maybe_install_npm_package markdownlint-cli2 markdownlint-cli2 "${MARKDOWNLINT_CLI2_VERSION}" +maybe_install_npm_package prettier prettier "${PRETTIER_VERSION}" maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" diff --git a/tests/data/cmd/config/config_3.json b/tests/data/cmd/config/config_3.json index 3517f16295177..a5e68a2ffd678 100644 --- a/tests/data/cmd/config/config_3.json +++ b/tests/data/cmd/config/config_3.json @@ -16,9 +16,7 @@ }, "transforms": { "transform0": { - "inputs": [ - "source0" - ], + "inputs": ["source0"], "drop_on_abort": false, "drop_on_error": false, "metric_tag_values": "single", @@ -29,9 +27,7 @@ }, "sinks": { "sink0": { - "inputs": [ - "transform0" - ], + "inputs": ["transform0"], "target": "stdout", "type": "console", "encoding": { diff --git a/tests/data/secret-backends/file-secrets.json b/tests/data/secret-backends/file-secrets.json index 8de9e8b673504..861c0cbdbc1a0 100644 --- a/tests/data/secret-backends/file-secrets.json +++ b/tests/data/secret-backends/file-secrets.json @@ -1 +1 @@ -{"ghi":"ghi.retrieved"} +{ "ghi": "ghi.retrieved" } diff --git a/tests/e2e/datadog-logs/config/compose.yaml b/tests/e2e/datadog-logs/config/compose.yaml index a144abed68cc5..774a47db4c729 100644 --- a/tests/e2e/datadog-logs/config/compose.yaml +++ b/tests/e2e/datadog-logs/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3.8' +version: "3.8" services: # Generates random log data for consumption by the custom Agent check @@ -28,11 +28,11 @@ services: depends_on: - fakeintake-agent environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent - - DD_ENABLE_PAYLOADS_EVENTS=false - - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - - DD_CONTAINER_EXCLUDE="name:.*" + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false + - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - type: bind @@ -58,11 +58,11 @@ services: depends_on: - vector environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent-vector - - DD_ENABLE_PAYLOADS_EVENTS=false - - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - - DD_CONTAINER_EXCLUDE="name:.*" + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false + - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - type: bind diff --git a/tests/e2e/datadog-logs/config/test.yaml b/tests/e2e/datadog-logs/config/test.yaml index 2ce4641b04e96..6582946497e7a 100644 --- a/tests/e2e/datadog-logs/config/test.yaml +++ b/tests/e2e/datadog-logs/config/test.yaml @@ -1,5 +1,5 @@ features: -- e2e-tests-datadog + - e2e-tests-datadog test: "e2e" @@ -7,22 +7,21 @@ test_filter: "datadog::logs::" runner: env: - EXPECTED_LOG_EVENTS: '1000' - VECTOR_RECEIVE_PORT: '8081' - FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' - FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' + EXPECTED_LOG_EVENTS: "1000" + VECTOR_RECEIVE_PORT: "8081" + FAKE_INTAKE_AGENT_ENDPOINT: "http://fakeintake-agent:80" + FAKE_INTAKE_VECTOR_ENDPOINT: "http://fakeintake-vector:80" matrix: # validate against the latest Agent nightly and also stable v6 and v7 - agent_version: ['latest', '6', '7'] - + agent_version: ["latest", "6", "7"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/sources/datadog_agent/**" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/logs/**" -- "src/sinks/util/**" -- "tests/e2e/datadog-logs/**" + - "src/common/datadog.rs" + - "src/sources/datadog_agent/**" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/logs/**" + - "src/sinks/util/**" + - "tests/e2e/datadog-logs/**" diff --git a/tests/e2e/datadog-logs/data/agent_only.yaml b/tests/e2e/datadog-logs/data/agent_only.yaml index b92e08db48f05..5f97845ce9b0e 100644 --- a/tests/e2e/datadog-logs/data/agent_only.yaml +++ b/tests/e2e/datadog-logs/data/agent_only.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'debug' +log_level: "debug" # disable bunch of stuff we don't need inventories_configuration_enabled: false @@ -29,4 +29,4 @@ logs_config: batch_wait: 1 # Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker -dd_url: 'http://fakeintake-agent:80' +dd_url: "http://fakeintake-agent:80" diff --git a/tests/e2e/datadog-logs/data/agent_vector.yaml b/tests/e2e/datadog-logs/data/agent_vector.yaml index 57ca8bff2d5e7..dd5432566d7c0 100644 --- a/tests/e2e/datadog-logs/data/agent_vector.yaml +++ b/tests/e2e/datadog-logs/data/agent_vector.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'debug' +log_level: "debug" # disable bunch of stuff we don't need inventories_configuration_enabled: false @@ -34,4 +34,4 @@ vector: url: "http://vector:8181" # Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker -dd_url: 'http://fakeintake-agent:80' +dd_url: "http://fakeintake-agent:80" diff --git a/tests/e2e/datadog-metrics/config/compose.yaml b/tests/e2e/datadog-metrics/config/compose.yaml index 84c0f89278863..6a317c156182f 100644 --- a/tests/e2e/datadog-metrics/config/compose.yaml +++ b/tests/e2e/datadog-metrics/config/compose.yaml @@ -1,12 +1,11 @@ -version: '3' +version: "3" services: - # Emits metrics to the Agent only path dogstatsd-client-agent: build: ../dogstatsd_client environment: - - STATSD_HOST=datadog-agent + - STATSD_HOST=datadog-agent depends_on: - datadog-agent @@ -14,7 +13,7 @@ services: dogstatsd-client-vector: build: ../dogstatsd_client environment: - - STATSD_HOST=datadog-agent-vector + - STATSD_HOST=datadog-agent-vector depends_on: - datadog-agent-vector @@ -24,8 +23,8 @@ services: depends_on: - fakeintake-agent environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent volumes: # The Agent config file - ../data/agent_only.yaml:/etc/datadog-agent/datadog.yaml @@ -36,8 +35,8 @@ services: depends_on: - vector environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent-vector + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector volumes: # The Agent config file - ../data/agent_vector.yaml:/etc/datadog-agent/datadog.yaml diff --git a/tests/e2e/datadog-metrics/config/test.yaml b/tests/e2e/datadog-metrics/config/test.yaml index 2b551be47c147..87b30504ace2f 100644 --- a/tests/e2e/datadog-metrics/config/test.yaml +++ b/tests/e2e/datadog-metrics/config/test.yaml @@ -1,28 +1,28 @@ features: -- e2e-tests-datadog + - e2e-tests-datadog test: "e2e" -test_filter: 'datadog::metrics::' +test_filter: "datadog::metrics::" runner: env: - VECTOR_RECEIVE_PORT: '8081' - FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' - FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' + VECTOR_RECEIVE_PORT: "8081" + FAKE_INTAKE_AGENT_ENDPOINT: "http://fakeintake-agent:80" + FAKE_INTAKE_VECTOR_ENDPOINT: "http://fakeintake-vector:80" matrix: # validate against the Agent latest nightly and also stable v6 and v7 - agent_version: ['latest', '6', '7'] + agent_version: ["latest", "6", "7"] # validate both series API versions - series_api_version: ['v1', 'v2'] + series_api_version: ["v1", "v2"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/sources/datadog_agent/**" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/metrics/**" -- "src/sinks/util/**" -- "tests/e2e/datadog-metrics/**" + - "src/common/datadog.rs" + - "src/sources/datadog_agent/**" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/metrics/**" + - "src/sinks/util/**" + - "tests/e2e/datadog-metrics/**" diff --git a/tests/e2e/datadog-metrics/data/agent_only.yaml b/tests/e2e/datadog-metrics/data/agent_only.yaml index 0ed0518c84b18..6b69a11cdf2cc 100644 --- a/tests/e2e/datadog-metrics/data/agent_only.yaml +++ b/tests/e2e/datadog-metrics/data/agent_only.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'warn' +log_level: "warn" # disable bunch of stuff we don't need inventories_configuration_enabled: false diff --git a/tests/e2e/datadog-metrics/data/agent_vector.yaml b/tests/e2e/datadog-metrics/data/agent_vector.yaml index c1566946a96b6..22709586bbd7b 100644 --- a/tests/e2e/datadog-metrics/data/agent_vector.yaml +++ b/tests/e2e/datadog-metrics/data/agent_vector.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'warn' +log_level: "warn" # disable bunch of stuff we don't need inventories_configuration_enabled: false diff --git a/tests/e2e/opentelemetry-logs/config/compose.yaml b/tests/e2e/opentelemetry-logs/config/compose.yaml index 62b3d48c44aa6..4066a553d2257 100644 --- a/tests/e2e/opentelemetry-logs/config/compose.yaml +++ b/tests/e2e/opentelemetry-logs/config/compose.yaml @@ -12,7 +12,7 @@ services: ports: - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317" - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318" - command: [ "--config=/etc/otelcol-contrib/config.yaml" ] + command: ["--config=/etc/otelcol-contrib/config.yaml"] logs-generator: container_name: logs-generator @@ -49,7 +49,7 @@ services: CONFIG_COLLECTOR_VERSION: ${CONFIG_COLLECTOR_VERSION} init: true user: "0:0" # test only, override special user with root - command: [ "--config", "/etc/otelcol-contrib/config.yaml" ] + command: ["--config", "/etc/otelcol-contrib/config.yaml"] volumes: - type: bind source: ../data/collector-sink.yaml @@ -76,7 +76,7 @@ services: environment: - VECTOR_LOG=${VECTOR_LOG:-info} - FEATURES=e2e-tests-opentelemetry - command: [ "vector", "-c", "/etc/vector/vector.yaml" ] + command: ["vector", "-c", "/etc/vector/vector.yaml"] volumes: vector_target: diff --git a/tests/e2e/opentelemetry-logs/config/test.yaml b/tests/e2e/opentelemetry-logs/config/test.yaml index 54aa7063e632c..e1c5cb85267d2 100644 --- a/tests/e2e/opentelemetry-logs/config/test.yaml +++ b/tests/e2e/opentelemetry-logs/config/test.yaml @@ -8,14 +8,14 @@ test_filter: "opentelemetry::logs::" runner: needs_docker_socket: true env: - OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' - OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' - OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + OTEL_COLLECTOR_SOURCE_GRPC_PORT: "4317" + OTEL_COLLECTOR_SOURCE_HTTP_PORT: "4318" + OTEL_COLLECTOR_SINK_HTTP_PORT: "5318" matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use - collector_version: [ 'latest' ] - vector_config: [ 'vector_default.yaml', 'vector_otlp.yaml' ] + collector_version: ["latest"] + vector_config: ["vector_default.yaml", "vector_otlp.yaml"] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/tests/e2e/opentelemetry-logs/data/collector-sink.yaml b/tests/e2e/opentelemetry-logs/data/collector-sink.yaml index e22f62f91d558..1da64319f6926 100644 --- a/tests/e2e/opentelemetry-logs/data/collector-sink.yaml +++ b/tests/e2e/opentelemetry-logs/data/collector-sink.yaml @@ -5,10 +5,10 @@ receivers: endpoint: "0.0.0.0:5318" processors: - batch: { } + batch: {} exporters: - debug: { } + debug: {} file: path: /output/opentelemetry-logs/collector-file-exporter.log rotation: @@ -18,9 +18,9 @@ exporters: service: pipelines: logs: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, file ] + receivers: [otlp] + processors: [batch] + exporters: [debug, file] telemetry: logs: level: "debug" diff --git a/tests/e2e/opentelemetry-logs/data/collector-source.yaml b/tests/e2e/opentelemetry-logs/data/collector-source.yaml index 1a41636209e78..d3f8427a47f33 100644 --- a/tests/e2e/opentelemetry-logs/data/collector-source.yaml +++ b/tests/e2e/opentelemetry-logs/data/collector-source.yaml @@ -6,7 +6,7 @@ receivers: grpc: processors: - batch: { } + batch: {} exporters: otlp/grpc: @@ -17,16 +17,16 @@ exporters: endpoint: http://vector:4318 tls: insecure: true - debug: { } + debug: {} service: pipelines: logs: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, otlp/grpc, otlphttp/vector ] + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/grpc, otlphttp/vector] metrics: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug ] + receivers: [otlp] + processors: [batch] + exporters: [debug] diff --git a/tests/e2e/opentelemetry-logs/data/vector_default.yaml b/tests/e2e/opentelemetry-logs/data/vector_default.yaml index 1f3fd9b119d09..b8e3f6ef9dad3 100644 --- a/tests/e2e/opentelemetry-logs/data/vector_default.yaml +++ b/tests/e2e/opentelemetry-logs/data/vector_default.yaml @@ -46,10 +46,9 @@ transforms: ] } - sinks: otel_sink: - inputs: [ "remap_otel" ] + inputs: ["remap_otel"] type: opentelemetry protocol: type: http diff --git a/tests/e2e/opentelemetry-metrics/config/compose.yaml b/tests/e2e/opentelemetry-metrics/config/compose.yaml index fba9c2a19b247..44d17475c55b2 100644 --- a/tests/e2e/opentelemetry-metrics/config/compose.yaml +++ b/tests/e2e/opentelemetry-metrics/config/compose.yaml @@ -12,7 +12,7 @@ services: ports: - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317" - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318" - command: [ "--config=/etc/otelcol-contrib/config.yaml" ] + command: ["--config=/etc/otelcol-contrib/config.yaml"] metrics-generator: container_name: metrics-generator @@ -86,7 +86,7 @@ services: CONFIG_COLLECTOR_VERSION: ${CONFIG_COLLECTOR_VERSION} init: true user: "0:0" # test only, override special user with root - command: [ "--config", "/etc/otelcol-contrib/config.yaml" ] + command: ["--config", "/etc/otelcol-contrib/config.yaml"] volumes: - type: bind source: ../data/collector-sink.yaml @@ -113,7 +113,7 @@ services: environment: - VECTOR_LOG=${VECTOR_LOG:-info} - FEATURES=e2e-tests-opentelemetry - command: [ "vector", "-c", "/etc/vector/vector.yaml" ] + command: ["vector", "-c", "/etc/vector/vector.yaml"] volumes: vector_target: diff --git a/tests/e2e/opentelemetry-metrics/config/test.yaml b/tests/e2e/opentelemetry-metrics/config/test.yaml index 507cddfd2b21d..4d5b0bf4d5256 100644 --- a/tests/e2e/opentelemetry-metrics/config/test.yaml +++ b/tests/e2e/opentelemetry-metrics/config/test.yaml @@ -8,13 +8,13 @@ test_filter: "opentelemetry::metrics::" runner: needs_docker_socket: true env: - OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' - OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' - OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + OTEL_COLLECTOR_SOURCE_GRPC_PORT: "4317" + OTEL_COLLECTOR_SOURCE_HTTP_PORT: "4318" + OTEL_COLLECTOR_SINK_HTTP_PORT: "5318" matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use - collector_version: [ 'latest' ] + collector_version: ["latest"] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml b/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml index 120b29ff8e20f..6fb3eb25e1522 100644 --- a/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml +++ b/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml @@ -5,10 +5,10 @@ receivers: endpoint: "0.0.0.0:5318" processors: - batch: { } + batch: {} exporters: - debug: { } + debug: {} file: path: /output/opentelemetry-metrics/collector-file-exporter.log rotation: @@ -18,9 +18,9 @@ exporters: service: pipelines: metrics: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, file ] + receivers: [otlp] + processors: [batch] + exporters: [debug, file] telemetry: logs: level: "debug" diff --git a/tests/e2e/opentelemetry-metrics/data/collector-source.yaml b/tests/e2e/opentelemetry-metrics/data/collector-source.yaml index 6f1ba27560e87..bf40ac05aa78c 100644 --- a/tests/e2e/opentelemetry-metrics/data/collector-source.yaml +++ b/tests/e2e/opentelemetry-metrics/data/collector-source.yaml @@ -6,7 +6,7 @@ receivers: grpc: processors: - batch: { } + batch: {} exporters: otlp/grpc: @@ -17,11 +17,11 @@ exporters: endpoint: http://vector:4318 tls: insecure: true - debug: { } + debug: {} service: pipelines: metrics: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, otlp/grpc, otlphttp/vector ] + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/grpc, otlphttp/vector] diff --git a/tests/integration/amqp/config/compose.yaml b/tests/integration/amqp/config/compose.yaml index 3eb859f93aa60..643f0b13f083f 100644 --- a/tests/integration/amqp/config/compose.yaml +++ b/tests/integration/amqp/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: rabbitmq: diff --git a/tests/integration/amqp/config/test.yaml b/tests/integration/amqp/config/test.yaml index 94c080a18559d..40a43748cd12d 100644 --- a/tests/integration/amqp/config/test.yaml +++ b/tests/integration/amqp/config/test.yaml @@ -1,17 +1,17 @@ features: -- amqp-integration-tests + - amqp-integration-tests -test_filter: '::amqp::' +test_filter: "::amqp::" matrix: - version: ['3.8'] + version: ["3.8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/amqp.rs" -- "src/internal_events/amqp.rs" -- "src/sinks/amqp/**" -- "src/sources/amqp.rs" -- "src/sources/util/**" -- "src/sinks/util/**" + - "src/amqp.rs" + - "src/internal_events/amqp.rs" + - "src/sinks/amqp/**" + - "src/sources/amqp.rs" + - "src/sources/util/**" + - "src/sinks/util/**" diff --git a/tests/integration/appsignal/config/test.yaml b/tests/integration/appsignal/config/test.yaml index d110306916df0..da826955f726a 100644 --- a/tests/integration/appsignal/config/test.yaml +++ b/tests/integration/appsignal/config/test.yaml @@ -1,7 +1,7 @@ features: -- appsignal-integration-tests + - appsignal-integration-tests -test_filter: '::appsignal::integration_tests::' +test_filter: "::appsignal::integration_tests::" runner: env: @@ -13,5 +13,5 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/appsignal/**" -- "src/sinks/util/**" + - "src/sinks/appsignal/**" + - "src/sinks/util/**" diff --git a/tests/integration/aws/config/compose.yaml b/tests/integration/aws/config/compose.yaml index 4fbbd62f6e120..a468d8e4c0a96 100644 --- a/tests/integration/aws/config/compose.yaml +++ b/tests/integration/aws/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: mock-ec2-metadata: @@ -6,15 +6,15 @@ services: mock-localstack: image: docker.io/localstack/localstack@sha256:46302bcb91a7e8008e6394be8afafdbfa40fb77a54d4046a38be35992042d5de environment: - - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs + - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs mock-ecs: image: docker.io/amazon/amazon-ecs-local-container-endpoints:latest environment: - # https://github.com/vectordotdev/vector/issues/24687 - - DOCKER_API_VERSION=1.44 + # https://github.com/vectordotdev/vector/issues/24687 + - DOCKER_API_VERSION=1.44 volumes: - - $DOCKER_SOCKET:/var/run/docker.sock - - $HOME/.aws/:/home/.aws/ + - $DOCKER_SOCKET:/var/run/docker.sock + - $HOME/.aws/:/home/.aws/ networks: default: diff --git a/tests/integration/aws/config/test.yaml b/tests/integration/aws/config/test.yaml index 956f724978fa3..d27214d9499de 100644 --- a/tests/integration/aws/config/test.yaml +++ b/tests/integration/aws/config/test.yaml @@ -1,5 +1,5 @@ features: -- aws-integration-tests + - aws-integration-tests test_filter: ::aws_ @@ -22,11 +22,11 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/aws/**" -- "src/internal_events/aws_*" -- "src/sources/aws_*/**" -- "src/sources/util/**" -- "src/sinks/aws_*/**" -- "src/sinks/util/**" -- "src/transforms/aws_*" -- "tests/integration/aws/**" + - "src/aws/**" + - "src/internal_events/aws_*" + - "src/sources/aws_*/**" + - "src/sources/util/**" + - "src/sinks/aws_*/**" + - "src/sinks/util/**" + - "src/transforms/aws_*" + - "tests/integration/aws/**" diff --git a/tests/integration/axiom/config/test.yaml b/tests/integration/axiom/config/test.yaml index 934bd739eeaca..ed542ece401d7 100644 --- a/tests/integration/axiom/config/test.yaml +++ b/tests/integration/axiom/config/test.yaml @@ -1,7 +1,7 @@ features: -- axiom-integration-tests + - axiom-integration-tests -test_filter: '::axiom::' +test_filter: "::axiom::" runner: env: @@ -16,6 +16,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/axiom.rs" -- "src/sinks/util/**" -- "tests/integration/axiom/**" + - "src/sinks/axiom.rs" + - "src/sinks/util/**" + - "tests/integration/axiom/**" diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index 933a72d235a24..fe3541bf2a07c 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -1,11 +1,11 @@ -version: '3' +version: "3" services: local-azure-blob: image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} command: azurite --blobHost 0.0.0.0 --loose volumes: - - /var/run:/var/run + - /var/run:/var/run networks: default: diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index 191bea7256cea..da00ceed79ad4 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -1,5 +1,5 @@ features: -- azure-integration-tests + - azure-integration-tests test_filter: ::azure_ @@ -9,11 +9,11 @@ env: LOGSTASH_ADDRESS: 0.0.0.0:8081 matrix: - version: [ 3.35.0 ] + version: [3.35.0] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/azure_**" -- "src/sinks/util/**" -- "tests/integration/azure/**" + - "src/sinks/azure_**" + - "src/sinks/util/**" + - "tests/integration/azure/**" diff --git a/tests/integration/clickhouse/config/compose.yaml b/tests/integration/clickhouse/config/compose.yaml index fd823897ec4c2..e861c92c736f0 100644 --- a/tests/integration/clickhouse/config/compose.yaml +++ b/tests/integration/clickhouse/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: clickhouse: diff --git a/tests/integration/clickhouse/config/test.yaml b/tests/integration/clickhouse/config/test.yaml index 3418d810fa638..3f88eedf5a654 100644 --- a/tests/integration/clickhouse/config/test.yaml +++ b/tests/integration/clickhouse/config/test.yaml @@ -1,17 +1,17 @@ features: -- clickhouse-integration-tests + - clickhouse-integration-tests -test_filter: '::clickhouse::' +test_filter: "::clickhouse::" env: CLICKHOUSE_ADDRESS: http://clickhouse:8123 matrix: - version: ['23'] + version: ["23"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/clickhouse/**" -- "src/sinks/util/**" -- "tests/integration/clickhouse/**" + - "src/sinks/clickhouse/**" + - "src/sinks/util/**" + - "tests/integration/clickhouse/**" diff --git a/tests/integration/databend/config/compose.yaml b/tests/integration/databend/config/compose.yaml index 213fb20af70fe..e98305ba2ea01 100644 --- a/tests/integration/databend/config/compose.yaml +++ b/tests/integration/databend/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: minio: diff --git a/tests/integration/databend/config/test.yaml b/tests/integration/databend/config/test.yaml index eaa9f5ad50d86..0ec4b2c9d57f4 100644 --- a/tests/integration/databend/config/test.yaml +++ b/tests/integration/databend/config/test.yaml @@ -1,18 +1,18 @@ features: -- databend-integration-tests + - databend-integration-tests -test_filter: '::databend::' +test_filter: "::databend::" runner: env: DATABEND_ENDPOINT: databend://vector:vector@databend:8000?sslmode=disable&presign=detect matrix: - version: ['latest'] + version: ["latest"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/databend/**" -- "src/sinks/util/**" -- "tests/integration/databend/**" + - "src/sinks/databend/**" + - "src/sinks/util/**" + - "tests/integration/databend/**" diff --git a/tests/integration/datadog-agent/config/compose.yaml b/tests/integration/datadog-agent/config/compose.yaml index 4ccf95ff24e00..c63089ac620a9 100644 --- a/tests/integration/datadog-agent/config/compose.yaml +++ b/tests/integration/datadog-agent/config/compose.yaml @@ -1,32 +1,32 @@ -version: '3' +version: "3" services: datadog-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=false - - DD_LOGS_ENABLED=true - - DD_LOGS_CONFIG_LOGS_DD_URL=runner:8080 - - DD_LOGS_CONFIG_LOGS_NO_SSL=true - - DD_LOGS_CONFIG_USE_HTTP=true - - DD_HEALTH_PORT=8182 - - DD_CMD_PORT=5001 - - DD_USE_DOGSTATSD=false - - DD_HOSTNAME=datadog-agent - - DD_SERIALIZER_COMPRESSOR_KIND=zstd + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=false + - DD_LOGS_ENABLED=true + - DD_LOGS_CONFIG_LOGS_DD_URL=runner:8080 + - DD_LOGS_CONFIG_LOGS_NO_SSL=true + - DD_LOGS_CONFIG_USE_HTTP=true + - DD_HEALTH_PORT=8182 + - DD_CMD_PORT=5001 + - DD_USE_DOGSTATSD=false + - DD_HOSTNAME=datadog-agent + - DD_SERIALIZER_COMPRESSOR_KIND=zstd volumes: - - ../data/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml + - ../data/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml datadog-trace-agent: image: docker.io/datadog/agent:7.31.0 environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=true - - DD_APM_DD_URL=http://runner:8081 - - DD_HEALTH_PORT=8183 - - DD_CMD_PORT=5002 - - DD_USE_DOGSTATSD=false - - DD_HOSTNAME=datadog-trace-agent + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=true + - DD_APM_DD_URL=http://runner:8081 + - DD_HEALTH_PORT=8183 + - DD_CMD_PORT=5002 + - DD_USE_DOGSTATSD=false + - DD_HOSTNAME=datadog-trace-agent networks: default: diff --git a/tests/integration/datadog-agent/config/test.yaml b/tests/integration/datadog-agent/config/test.yaml index c1d3ec50d7592..353c1c9e02188 100644 --- a/tests/integration/datadog-agent/config/test.yaml +++ b/tests/integration/datadog-agent/config/test.yaml @@ -1,7 +1,7 @@ features: -- datadog-agent-integration-tests + - datadog-agent-integration-tests -test_filter: 'sources::datadog_agent::integration_tests::' +test_filter: "sources::datadog_agent::integration_tests::" env: AGENT_ADDRESS: datadog-agent:8181 @@ -11,12 +11,12 @@ env: TEST_DATADOG_API_KEY: matrix: - version: ['7'] + version: ["7"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sources/datadog_agent/**" -- "tests/integration/datadog-agent/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sources/datadog_agent/**" + - "tests/integration/datadog-agent/**" diff --git a/tests/integration/datadog-logs/config/test.yaml b/tests/integration/datadog-logs/config/test.yaml index f10db04ef5661..3e33d0ffc8922 100644 --- a/tests/integration/datadog-logs/config/test.yaml +++ b/tests/integration/datadog-logs/config/test.yaml @@ -1,7 +1,7 @@ features: -- datadog-logs-integration-tests + - datadog-logs-integration-tests -test_filter: '::datadog::logs::integration_tests::' +test_filter: "::datadog::logs::integration_tests::" runner: env: @@ -13,8 +13,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/logs/**" -- "src/sinks/util/**" -- "tests/integration/datadog-logs/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/logs/**" + - "src/sinks/util/**" + - "tests/integration/datadog-logs/**" diff --git a/tests/integration/datadog-metrics/config/test.yaml b/tests/integration/datadog-metrics/config/test.yaml index 55282a361eccb..a0cdf30b036a1 100644 --- a/tests/integration/datadog-metrics/config/test.yaml +++ b/tests/integration/datadog-metrics/config/test.yaml @@ -1,7 +1,7 @@ features: -- datadog-metrics-integration-tests + - datadog-metrics-integration-tests -test_filter: '::datadog::metrics::integration_tests' +test_filter: "::datadog::metrics::integration_tests" runner: env: @@ -13,8 +13,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/metrics/**" -- "src/sinks/util/**" -- "tests/integration/datadog-metrics/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/metrics/**" + - "src/sinks/util/**" + - "tests/integration/datadog-metrics/**" diff --git a/tests/integration/datadog-traces/config/compose.yaml b/tests/integration/datadog-traces/config/compose.yaml index f94ee0b53d089..847db7b3c1387 100644 --- a/tests/integration/datadog-traces/config/compose.yaml +++ b/tests/integration/datadog-traces/config/compose.yaml @@ -1,34 +1,34 @@ -version: '3' +version: "3" services: datadog-trace-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=true - - DD_APM_DD_URL=http://runner:8082 - - DD_HEALTH_PORT=8183 - - DD_CMD_PORT=5002 - - DD_USE_DOGSTATSD=false - - DD_APM_MAX_TPS=999999 - - DD_APM_ERROR_TPS=999999 - - DD_APM_MAX_MEMORY=0 - - DD_APM_MAX_CPU_PERCENT=0 - - DD_HOSTNAME=datadog-trace-agent + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=true + - DD_APM_DD_URL=http://runner:8082 + - DD_HEALTH_PORT=8183 + - DD_CMD_PORT=5002 + - DD_USE_DOGSTATSD=false + - DD_APM_MAX_TPS=999999 + - DD_APM_ERROR_TPS=999999 + - DD_APM_MAX_MEMORY=0 + - DD_APM_MAX_CPU_PERCENT=0 + - DD_HOSTNAME=datadog-trace-agent datadog-trace-agent-to-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=true - - DD_APM_DD_URL=http://runner:8081 - - DD_HEALTH_PORT=8183 - - DD_CMD_PORT=5002 - - DD_USE_DOGSTATSD=false - - DD_APM_MAX_TPS=999999 - - DD_APM_ERROR_TPS=999999 - - DD_APM_MAX_MEMORY=0 - - DD_APM_MAX_CPU_PERCENT=0 - - DD_HOSTNAME=datadog-trace-agent-to-vector + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=true + - DD_APM_DD_URL=http://runner:8081 + - DD_HEALTH_PORT=8183 + - DD_CMD_PORT=5002 + - DD_USE_DOGSTATSD=false + - DD_APM_MAX_TPS=999999 + - DD_APM_ERROR_TPS=999999 + - DD_APM_MAX_MEMORY=0 + - DD_APM_MAX_CPU_PERCENT=0 + - DD_HOSTNAME=datadog-trace-agent-to-vector networks: default: diff --git a/tests/integration/datadog-traces/config/test.yaml b/tests/integration/datadog-traces/config/test.yaml index 2a3f162e61b30..7f72063f617c7 100644 --- a/tests/integration/datadog-traces/config/test.yaml +++ b/tests/integration/datadog-traces/config/test.yaml @@ -1,14 +1,14 @@ features: -- datadog-traces-integration-tests + - datadog-traces-integration-tests -test_filter: '::datadog::traces::' +test_filter: "::datadog::traces::" env: - AGENT_PORT: '8082' + AGENT_PORT: "8082" VECTOR_LOG: ${VECTOR_LOG} TRACE_AGENT_TO_VECTOR_URL: http://datadog-trace-agent-to-vector:8126/v0.3/traces TRACE_AGENT_URL: http://datadog-trace-agent:8126/v0.3/traces - VECTOR_PORT: '8081' + VECTOR_PORT: "8081" TEST_DATADOG_API_KEY: matrix: @@ -17,8 +17,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/traces/**" -- "src/sinks/util/**" -- "tests/integration/datadog-traces/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/traces/**" + - "src/sinks/util/**" + - "tests/integration/datadog-traces/**" diff --git a/tests/integration/dnstap/config/test.yaml b/tests/integration/dnstap/config/test.yaml index ed0de1d3ea411..8ff55500cbf59 100644 --- a/tests/integration/dnstap/config/test.yaml +++ b/tests/integration/dnstap/config/test.yaml @@ -1,7 +1,7 @@ features: -- dnstap-integration-tests + - dnstap-integration-tests -test_filter: '::dnstap::' +test_filter: "::dnstap::" runner: env: @@ -11,12 +11,12 @@ runner: dnstap_dnstap-sockets: /run/bind/socket matrix: - version: ['latest'] + version: ["latest"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/dnstap.rs" -- "src/sources/dnstap/**" -- "src/sources/util/**" -- "tests/integration/dnstap/**" + - "src/internal_events/dnstap.rs" + - "src/sources/dnstap/**" + - "src/sources/util/**" + - "tests/integration/dnstap/**" diff --git a/tests/integration/docker-logs/config/test.yaml b/tests/integration/docker-logs/config/test.yaml index 0ac64071bd516..2ddd643652e9c 100644 --- a/tests/integration/docker-logs/config/test.yaml +++ b/tests/integration/docker-logs/config/test.yaml @@ -1,5 +1,5 @@ features: -- docker-logs-integration-tests + - docker-logs-integration-tests test_filter: "::docker_logs::" @@ -12,8 +12,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/docker.rs" -- "src/internal_events/docker_logs.rs" -- "src/sources/docker_logs/**" -- "src/sources/util/**" -- "tests/integration/docker-logs/**" + - "src/docker.rs" + - "src/internal_events/docker_logs.rs" + - "src/sources/docker_logs/**" + - "src/sources/util/**" + - "tests/integration/docker-logs/**" diff --git a/tests/integration/doris/config/test.yaml b/tests/integration/doris/config/test.yaml index 799a369d19a2f..e09e964991eb4 100644 --- a/tests/integration/doris/config/test.yaml +++ b/tests/integration/doris/config/test.yaml @@ -1,19 +1,19 @@ features: -- sinks-doris -- doris-integration-tests + - sinks-doris + - doris-integration-tests -test_filter: 'sinks::doris::integration_test::' +test_filter: "sinks::doris::integration_test::" env: DORIS_ADDRESS: http://doris:8040 matrix: - version: ['2.1.7'] + version: ["2.1.7"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/doris.rs" -- "src/sinks/doris/**" -- "src/sinks/util/**" -- "tests/integration/doris/**" + - "src/internal_events/doris.rs" + - "src/sinks/doris/**" + - "src/sinks/util/**" + - "tests/integration/doris/**" diff --git a/tests/integration/elasticsearch/config/compose.yaml b/tests/integration/elasticsearch/config/compose.yaml index 4dcb6636bf4cb..cb6d0bcc2f392 100644 --- a/tests/integration/elasticsearch/config/compose.yaml +++ b/tests/integration/elasticsearch/config/compose.yaml @@ -1,30 +1,30 @@ -version: '3' +version: "3" services: localstack: image: docker.io/localstack/localstack@sha256:f21f1fc770ee4bfd5012afdc902154c56b7fb18c14cf672de151b65569c8251e environment: - - SERVICES=elasticsearch:4571 + - SERVICES=elasticsearch:4571 elasticsearch: image: docker.io/elasticsearch:${CONFIG_VERSION} environment: - - discovery.type=single-node - - ES_JAVA_OPTS=-Xms400m -Xmx400m + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms400m -Xmx400m elasticsearch-secure: image: docker.io/elasticsearch:${CONFIG_VERSION} environment: - - discovery.type=single-node - - xpack.security.enabled=true - - xpack.security.http.ssl.enabled=true - - xpack.security.http.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem - - xpack.security.http.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem - - xpack.security.transport.ssl.enabled=true - - xpack.security.transport.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem - - xpack.security.transport.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem - - ELASTIC_PASSWORD=vector - - ES_JAVA_OPTS=-Xms400m -Xmx400m + - discovery.type=single-node + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem + - xpack.security.http.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem + - xpack.security.transport.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem + - ELASTIC_PASSWORD=vector + - ES_JAVA_OPTS=-Xms400m -Xmx400m volumes: - - ../../../data/ca:/usr/share/elasticsearch/config/certs:ro + - ../../../data/ca:/usr/share/elasticsearch/config/certs:ro networks: default: diff --git a/tests/integration/elasticsearch/config/test.yaml b/tests/integration/elasticsearch/config/test.yaml index cdf1eedbbb4a7..984932cb08540 100644 --- a/tests/integration/elasticsearch/config/test.yaml +++ b/tests/integration/elasticsearch/config/test.yaml @@ -1,7 +1,7 @@ features: -- es-integration-tests + - es-integration-tests -test_filter: '::elasticsearch::integration_tests::' +test_filter: "::elasticsearch::integration_tests::" env: AWS_ACCESS_KEY_ID: dummy @@ -16,6 +16,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/elasticsearch/**" -- "src/sinks/util/**" -- "tests/integration/elasticsearch/**" + - "src/sinks/elasticsearch/**" + - "src/sinks/util/**" + - "tests/integration/elasticsearch/**" diff --git a/tests/integration/eventstoredb/config/compose.yaml b/tests/integration/eventstoredb/config/compose.yaml index 01e6a14ea4117..76d9c45a8528b 100644 --- a/tests/integration/eventstoredb/config/compose.yaml +++ b/tests/integration/eventstoredb/config/compose.yaml @@ -1,11 +1,11 @@ -version: '3' +version: "3" services: eventstoredb: image: docker.io/eventstore/eventstore:${CONFIG_VERSION} command: --insecure --stats-period-sec=1 volumes: - - ../../../data:/etc/vector:ro + - ../../../data:/etc/vector:ro networks: default: diff --git a/tests/integration/eventstoredb/config/test.yaml b/tests/integration/eventstoredb/config/test.yaml index 2dab755bfd032..60c6d8e148cc8 100644 --- a/tests/integration/eventstoredb/config/test.yaml +++ b/tests/integration/eventstoredb/config/test.yaml @@ -1,7 +1,7 @@ features: -- eventstoredb_metrics-integration-tests + - eventstoredb_metrics-integration-tests -test_filter: '::eventstoredb_metrics::' +test_filter: "::eventstoredb_metrics::" matrix: version: ["24.2", "24.6", "latest"] @@ -9,7 +9,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/eventstoredb_metrics.rs" -- "src/sources/eventstoredb_metrics/**" -- "src/sources/util/**" -- "tests/integration/eventstoredb/**" + - "src/internal_events/eventstoredb_metrics.rs" + - "src/sources/eventstoredb_metrics/**" + - "src/sources/util/**" + - "tests/integration/eventstoredb/**" diff --git a/tests/integration/fluent/config/test.yaml b/tests/integration/fluent/config/test.yaml index f5f977aa1f637..41151d5abd8af 100644 --- a/tests/integration/fluent/config/test.yaml +++ b/tests/integration/fluent/config/test.yaml @@ -1,5 +1,5 @@ features: -- fluent-integration-tests + - fluent-integration-tests test_filter: "::fluent::" @@ -14,7 +14,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/fluent.rs" -- "src/sources/fluent/**" -- "src/sources/util/**" -- "tests/integration/fluent/**" + - "src/internal_events/fluent.rs" + - "src/sources/fluent/**" + - "src/sources/util/**" + - "tests/integration/fluent/**" diff --git a/tests/integration/gcp/config/compose.yaml b/tests/integration/gcp/config/compose.yaml index f317c23a5604c..d911f9fad8872 100644 --- a/tests/integration/gcp/config/compose.yaml +++ b/tests/integration/gcp/config/compose.yaml @@ -1,20 +1,20 @@ -version: '3' +version: "3" services: gcloud-pubsub: image: docker.io/messagebird/gcloud-pubsub-emulator:${CONFIG_VERSION} environment: - - PUBSUB_PROJECT1=testproject,topic1:subscription1 - - PUBSUB_PROJECT2=sourceproject,topic2:subscription2 + - PUBSUB_PROJECT1=testproject,topic1:subscription1 + - PUBSUB_PROJECT2=sourceproject,topic2:subscription2 chronicle-emulator: image: docker.io/timberio/chronicle-emulator:${CONFIG_VERSION} ports: - - 3000:3000 + - 3000:3000 volumes: - - ./public.pem:/public.pem:ro + - ./public.pem:/public.pem:ro command: - - -p - - /public.pem + - -p + - /public.pem networks: default: diff --git a/tests/integration/gcp/config/invalidauth.json b/tests/integration/gcp/config/invalidauth.json index 7126a1b4afbf6..0963bfc473027 100644 --- a/tests/integration/gcp/config/invalidauth.json +++ b/tests/integration/gcp/config/invalidauth.json @@ -2,7 +2,7 @@ "type": "service_account", "project_id": "chronicle-test", "private_key_id": "1", - "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIICXgIBAAKBgQDouHdVDVz0/M6PGe60Kf/g0nyOxCvbZgiUAZNzFimXDU+RpZ54\n6/oETl6VpRkbp8a4Xb8avll2lsamdHvGcsgnjJXdpp7LfWYLqHEpn0/XFM+womXg\nvglWCDwAsXmrmwpZKEC82mmyFigheyPA/sfuN6z+wa7P5B65xzIdDQX7nQIDAQAB\nAoGBANID/rUDrTrtll8v8Oon6OH0MjIIuOdzKhSfY3h9rKTDf2YaB2xq0KLoMpVr\ne8AoZb5l45t34naR1M3M2xKY7SSDAVJFfg/3Vxeot86DQ23IGLXj7LnNxXnvklXa\nEXaD8LNz/MXxS7/Lu0R+lEtjEkf23+BRb11fL6Q/EDToNHnhAkEA/FnwHhKMc/Bm\nXsS8bENuZP3SV2v7TU6MFTtXJFmsoZBxHnsM8UUi0gq9gBnApmdhy7v2N/Mv9gFI\nviSdr7vm1QJBAOwV3cHAciRHVK71TweOWIJKZBM9ZVut0VDs5GrBYZxGMBiOr3BI\ns7+0ugTKxVimuei6c0KNXw1kg3Vtc5+utakCQQDklAbXBpAomJHxt5zBKBc/7VXx\nEANyk/p5ZOXbLEsdkXuVU3p2tNwEi+v4s9r4H97Kr3goV+SSnbkpWntm6fn9AkBn\nFnE7rlXpA4C12QYGTaDWW7dxM0j0DGUvChH/j6uYuok73+o5hHWAy2DCwOwFduAN\nAIVd1S9hQLeqaf2oB3jpAkEAnRT+bAlMjtUOBO6XPNO4IbYwWJvGMcIEO7zu6AdB\nPJy3/U+bLimxFuYdrs6SnIHIUVdl35AlckHqzT54a5YKqQ==\n-----END RSA PRIVATE KEY-----", + "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIICXgIBAAKBgQDouHdVDVz0/M6PGe60Kf/g0nyOxCvbZgiUAZNzFimXDU+RpZ54\n6/oETl6VpRkbp8a4Xb8avll2lsamdHvGcsgnjJXdpp7LfWYLqHEpn0/XFM+womXg\nvglWCDwAsXmrmwpZKEC82mmyFigheyPA/sfuN6z+wa7P5B65xzIdDQX7nQIDAQAB\nAoGBANID/rUDrTrtll8v8Oon6OH0MjIIuOdzKhSfY3h9rKTDf2YaB2xq0KLoMpVr\ne8AoZb5l45t34naR1M3M2xKY7SSDAVJFfg/3Vxeot86DQ23IGLXj7LnNxXnvklXa\nEXaD8LNz/MXxS7/Lu0R+lEtjEkf23+BRb11fL6Q/EDToNHnhAkEA/FnwHhKMc/Bm\nXsS8bENuZP3SV2v7TU6MFTtXJFmsoZBxHnsM8UUi0gq9gBnApmdhy7v2N/Mv9gFI\nviSdr7vm1QJBAOwV3cHAciRHVK71TweOWIJKZBM9ZVut0VDs5GrBYZxGMBiOr3BI\ns7+0ugTKxVimuei6c0KNXw1kg3Vtc5+utakCQQDklAbXBpAomJHxt5zBKBc/7VXx\nEANyk/p5ZOXbLEsdkXuVU3p2tNwEi+v4s9r4H97Kr3goV+SSnbkpWntm6fn9AkBn\nFnE7rlXpA4C12QYGTaDWW7dxM0j0DGUvChH/j6uYuok73+o5hHWAy2DCwOwFduAN\nAIVd1S9hQLeqaf2oB3jpAkEAnRT+bAlMjtUOBO6XPNO4IbYwWJvGMcIEO7zu6AdB\nPJy3/U+bLimxFuYdrs6SnIHIUVdl35AlckHqzT54a5YKqQ==\n-----END RSA PRIVATE KEY-----", "client_email": "test@test.com", "client_id": "1", "auth_uri": "http://chronicle-emulator:3000/o/oauth2/auth", diff --git a/tests/integration/gcp/config/test.yaml b/tests/integration/gcp/config/test.yaml index b700b8283f831..474826afd6aed 100644 --- a/tests/integration/gcp/config/test.yaml +++ b/tests/integration/gcp/config/test.yaml @@ -1,8 +1,8 @@ features: -- gcp-integration-tests -- chronicle-integration-tests + - gcp-integration-tests + - chronicle-integration-tests -test_filter: '::gcp' +test_filter: "::gcp" env: EMULATOR_ADDRESS: http://gcloud-pubsub:8681 @@ -14,10 +14,10 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/gcp_pubsub.rs" -- "src/sources/gcp_pubsub.rs" -- "src/sources/util/**" -- "src/sinks/gcp/**" -- "src/sinks/util/**" -- "src/gcp.rs" -- "tests/integration/gcp/**" + - "src/internal_events/gcp_pubsub.rs" + - "src/sources/gcp_pubsub.rs" + - "src/sources/util/**" + - "src/sinks/gcp/**" + - "src/sinks/util/**" + - "src/gcp.rs" + - "tests/integration/gcp/**" diff --git a/tests/integration/greptimedb/config/compose.yaml b/tests/integration/greptimedb/config/compose.yaml index 02a68364e7a31..01abc946761e1 100644 --- a/tests/integration/greptimedb/config/compose.yaml +++ b/tests/integration/greptimedb/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: greptimedb: diff --git a/tests/integration/greptimedb/config/test.yaml b/tests/integration/greptimedb/config/test.yaml index 17593b891df8c..2ec3984938617 100644 --- a/tests/integration/greptimedb/config/test.yaml +++ b/tests/integration/greptimedb/config/test.yaml @@ -1,7 +1,7 @@ features: -- greptimedb-integration-tests + - greptimedb-integration-tests -test_filter: '::greptimedb::' +test_filter: "::greptimedb::" runner: env: @@ -17,5 +17,5 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/greptimedb/**" -- "tests/integration/greptimedb/**" + - "src/sinks/greptimedb/**" + - "tests/integration/greptimedb/**" diff --git a/tests/integration/http-client/config/compose.yaml b/tests/integration/http-client/config/compose.yaml index 7ad5dfc47386b..91eaa066886eb 100644 --- a/tests/integration/http-client/config/compose.yaml +++ b/tests/integration/http-client/config/compose.yaml @@ -1,34 +1,34 @@ -version: '3' +version: "3" services: dufs: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: - - /data + - /data volumes: - - ../data/serve:/data + - ../data/serve:/data dufs-auth: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: - - -a - - "user:pass@/" - - --auth-method - - basic - - /data + - -a + - "user:pass@/" + - --auth-method + - basic + - /data volumes: - - ../data/serve:/data + - ../data/serve:/data dufs-https: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: - - --tls-cert - - /certs/ca.cert.pem - - --tls-key - - /certs/ca.key.pem - - /data + - --tls-cert + - /certs/ca.cert.pem + - --tls-key + - /certs/ca.key.pem + - /data volumes: - - ../data/serve:/data - - ../../../data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem - - ../../../data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem + - ../data/serve:/data + - ../../../data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem + - ../../../data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem networks: default: diff --git a/tests/integration/http-client/config/test.yaml b/tests/integration/http-client/config/test.yaml index a56570dc5cf81..bf52050571683 100644 --- a/tests/integration/http-client/config/test.yaml +++ b/tests/integration/http-client/config/test.yaml @@ -1,7 +1,7 @@ features: -- http-client-integration-tests + - http-client-integration-tests -test_filter: 'sources::http_client::' +test_filter: "sources::http_client::" env: DUFS_ADDRESS: http://dufs:5000 @@ -14,6 +14,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/http_client/**" -- "src/sources/util/**" -- "tests/integration/http-client/**" + - "src/sources/http_client/**" + - "src/sources/util/**" + - "tests/integration/http-client/**" diff --git a/tests/integration/http-client/data/serve/logs/json.json b/tests/integration/http-client/data/serve/logs/json.json index 9672096ad8861..9cc07fbc01e5d 100644 --- a/tests/integration/http-client/data/serve/logs/json.json +++ b/tests/integration/http-client/data/serve/logs/json.json @@ -1 +1 @@ -{ "foo" : "baz" } +{ "foo": "baz" } diff --git a/tests/integration/http-client/data/serve/metrics/native.json b/tests/integration/http-client/data/serve/metrics/native.json index 4aea8048614d2..76935bd2f32aa 100644 --- a/tests/integration/http-client/data/serve/metrics/native.json +++ b/tests/integration/http-client/data/serve/metrics/native.json @@ -1 +1 @@ -{ "metric" : { "name" : "a_metric", "kind" : "absolute" , "counter" : { "value" : 1 } } } +{ "metric": { "name": "a_metric", "kind": "absolute", "counter": { "value": 1 } } } diff --git a/tests/integration/http-client/data/serve/traces/native.json b/tests/integration/http-client/data/serve/traces/native.json index cc755b92f3bea..0d970b94d3e4f 100644 --- a/tests/integration/http-client/data/serve/traces/native.json +++ b/tests/integration/http-client/data/serve/traces/native.json @@ -1 +1 @@ -{ "trace" : { "name" : "a_trace", "foo" : 42 } } +{ "trace": { "name": "a_trace", "foo": 42 } } diff --git a/tests/integration/humio/config/compose.yaml b/tests/integration/humio/config/compose.yaml index bc1045d98445d..0fb6c2cb5ddbe 100644 --- a/tests/integration/humio/config/compose.yaml +++ b/tests/integration/humio/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: humio: diff --git a/tests/integration/humio/config/test.yaml b/tests/integration/humio/config/test.yaml index a8cdebeba3b19..9d068364bf917 100644 --- a/tests/integration/humio/config/test.yaml +++ b/tests/integration/humio/config/test.yaml @@ -1,7 +1,7 @@ features: -- humio-integration-tests + - humio-integration-tests -test_filter: '::humio::' +test_filter: "::humio::" runner: env: @@ -13,6 +13,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/humio/**" -- "src/sinks/util/**" -- "tests/integration/humio/**" + - "src/sinks/humio/**" + - "src/sinks/util/**" + - "tests/integration/humio/**" diff --git a/tests/integration/influxdb/config/compose.yaml b/tests/integration/influxdb/config/compose.yaml index b5c31c006ab06..09049db7c6a31 100644 --- a/tests/integration/influxdb/config/compose.yaml +++ b/tests/integration/influxdb/config/compose.yaml @@ -1,24 +1,24 @@ -version: '3' +version: "3" services: influxdb-v1: image: docker.io/influxdb:${CONFIG_VERSION} environment: - - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_REPORTING_DISABLED=true influxdb-v1-tls: image: docker.io/influxdb:${CONFIG_VERSION} environment: - - INFLUXDB_REPORTING_DISABLED=true - - INFLUXDB_HTTP_HTTPS_ENABLED=true - - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem - - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem + - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_HTTP_HTTPS_ENABLED=true + - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem + - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem volumes: - - ../../../data/ca:/etc/ssl:ro + - ../../../data/ca:/etc/ssl:ro influxdb-v2: image: docker.io/influxdb:2.0 command: influxd --reporting-disabled environment: - - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_REPORTING_DISABLED=true networks: default: diff --git a/tests/integration/influxdb/config/test.yaml b/tests/integration/influxdb/config/test.yaml index 35bf2f72691bf..d6965c383f24e 100644 --- a/tests/integration/influxdb/config/test.yaml +++ b/tests/integration/influxdb/config/test.yaml @@ -1,7 +1,7 @@ features: -- influxdb-integration-tests + - influxdb-integration-tests -test_filter: '::influxdb::' +test_filter: "::influxdb::" env: INFLUXDB_V1_HTTPS_ADDRESS: https://influxdb-v1-tls:8086 @@ -9,12 +9,12 @@ env: INFLUXDB_V2_ADDRESS: http://influxdb-v2:8086 matrix: - version: ['1.8'] + version: ["1.8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/influxdb.rs" -- "src/sinks/influxdb/**" -- "src/sinks/util/**" -- "tests/integration/influxdb/**" + - "src/internal_events/influxdb.rs" + - "src/sinks/influxdb/**" + - "src/sinks/util/**" + - "tests/integration/influxdb/**" diff --git a/tests/integration/kafka/config/compose.yaml b/tests/integration/kafka/config/compose.yaml index 81a35dd64ec94..66c16b7fc5f3a 100644 --- a/tests/integration/kafka/config/compose.yaml +++ b/tests/integration/kafka/config/compose.yaml @@ -1,39 +1,39 @@ -version: '3' +version: "3" services: zookeeper: image: docker.io/confluentinc/cp-zookeeper:${CONFIG_VERSION} ports: - - 2181:2181 + - 2181:2181 environment: - - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_CLIENT_PORT=2181 kafka: image: docker.io/confluentinc/cp-kafka:7.6.1 depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - ZOOKEEPER_SASL_ENABLED=false - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 - - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 - - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 - - KAFKA_SSL_CLIENT_AUTH=none - - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf - - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ZOOKEEPER_SASL_ENABLED=false + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 + - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 + - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 + - KAFKA_SSL_CLIENT_AUTH=none + - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf + - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN ports: - - 9091:9091 - - 9092:9092 - - 9093:9093 + - 9091:9091 + - 9092:9092 + - 9093:9093 volumes: - - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro - - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro - - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf + - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro + - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro + - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf networks: default: diff --git a/tests/integration/kafka/config/test.yaml b/tests/integration/kafka/config/test.yaml index 9619b4d7ea0f1..7f91f693b25dd 100644 --- a/tests/integration/kafka/config/test.yaml +++ b/tests/integration/kafka/config/test.yaml @@ -1,7 +1,7 @@ features: -- kafka-integration-tests + - kafka-integration-tests -test_filter: '::kafka::' +test_filter: "::kafka::" env: KAFKA_HOST: kafka @@ -12,10 +12,10 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/kafka.rs" -- "src/sinks/kafka/**" -- "src/sinks/util/**" -- "src/sources/kafka.rs" -- "src/sources/util/**" -- "src/kafka.rs" -- "tests/integration/kafka/**" + - "src/internal_events/kafka.rs" + - "src/sinks/kafka/**" + - "src/sinks/util/**" + - "src/sources/kafka.rs" + - "src/sources/util/**" + - "src/kafka.rs" + - "tests/integration/kafka/**" diff --git a/tests/integration/logstash/config/compose.yaml b/tests/integration/logstash/config/compose.yaml index af024d0855453..212ce4e1155be 100644 --- a/tests/integration/logstash/config/compose.yaml +++ b/tests/integration/logstash/config/compose.yaml @@ -1,17 +1,17 @@ -version: '3' +version: "3" services: beats-heartbeat: image: docker.elastic.co/beats/heartbeat:${CONFIG_VERSION} command: -environment=container -strict.perms=false volumes: - - ../data/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro + - ../data/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro logstash: image: docker.elastic.co/logstash/logstash:7.13.1 volumes: - - /dev/null:/usr/share/logstash/pipeline/logstash.yml - - ../../shared/data/host.docker.internal.crt:/tmp/logstash.crt - - ../data/logstash.conf:/usr/share/logstash/pipeline/logstash.conf + - /dev/null:/usr/share/logstash/pipeline/logstash.yml + - ../../shared/data/host.docker.internal.crt:/tmp/logstash.crt + - ../data/logstash.conf:/usr/share/logstash/pipeline/logstash.conf networks: default: diff --git a/tests/integration/logstash/config/test.yaml b/tests/integration/logstash/config/test.yaml index e0f26afbec7f7..490293912fbae 100644 --- a/tests/integration/logstash/config/test.yaml +++ b/tests/integration/logstash/config/test.yaml @@ -1,7 +1,7 @@ features: -- logstash-integration-tests + - logstash-integration-tests -test_filter: '::logstash::integration_tests::' +test_filter: "::logstash::integration_tests::" env: HEARTBEAT_ADDRESS: 0.0.0.0:8080 @@ -13,6 +13,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/logstash.rs" -- "src/sources/util/**" -- "tests/integration/logstash/**" + - "src/sources/logstash.rs" + - "src/sources/util/**" + - "tests/integration/logstash/**" diff --git a/tests/integration/logstash/data/heartbeat.yml b/tests/integration/logstash/data/heartbeat.yml index ee2a9c1bca24d..154f8cb971737 100644 --- a/tests/integration/logstash/data/heartbeat.yml +++ b/tests/integration/logstash/data/heartbeat.yml @@ -1,8 +1,8 @@ heartbeat.monitors: -- type: http - schedule: '@every 1s' - urls: - - https://google.com + - type: http + schedule: "@every 1s" + urls: + - https://google.com output.logstash: - hosts: ['runner:8080'] + hosts: ["runner:8080"] diff --git a/tests/integration/loki/config/compose.yaml b/tests/integration/loki/config/compose.yaml index 4350e809f8961..187c1c274bf71 100644 --- a/tests/integration/loki/config/compose.yaml +++ b/tests/integration/loki/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: loki: diff --git a/tests/integration/loki/config/test.yaml b/tests/integration/loki/config/test.yaml index 9a86d9ce6c29d..3d3c0a9c9902f 100644 --- a/tests/integration/loki/config/test.yaml +++ b/tests/integration/loki/config/test.yaml @@ -1,7 +1,7 @@ features: -- loki-integration-tests + - loki-integration-tests -test_filter: '::loki::' +test_filter: "::loki::" env: LOKI_ADDRESS: http://loki:3100 @@ -12,7 +12,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/loki.rs" -- "src/sinks/loki/**" -- "src/sinks/util/**" -- "tests/integration/loki/**" + - "src/internal_events/loki.rs" + - "src/sinks/loki/**" + - "src/sinks/util/**" + - "tests/integration/loki/**" diff --git a/tests/integration/mongodb/config/compose.yaml b/tests/integration/mongodb/config/compose.yaml index f7cd8b6517471..f06682b462763 100644 --- a/tests/integration/mongodb/config/compose.yaml +++ b/tests/integration/mongodb/config/compose.yaml @@ -1,35 +1,35 @@ -version: '3' +version: "3" services: mongodb-primary: image: docker.io/bitnamilegacy/mongodb:${CONFIG_VERSION} environment: - - MONGODB_ADVERTISED_HOSTNAME=mongodb-primary - - MONGODB_REPLICA_SET_MODE=primary - - MONGODB_ROOT_PASSWORD=toor - - MONGODB_REPLICA_SET_KEY=vector + - MONGODB_ADVERTISED_HOSTNAME=mongodb-primary + - MONGODB_REPLICA_SET_MODE=primary + - MONGODB_ROOT_PASSWORD=toor + - MONGODB_REPLICA_SET_KEY=vector mongodb-secondary: image: docker.io/bitnamilegacy/mongodb:${CONFIG_VERSION} depends_on: - - mongodb-primary + - mongodb-primary environment: - - MONGODB_ADVERTISED_HOSTNAME=mongodb-secondary - - MONGODB_REPLICA_SET_MODE=secondary - - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary - - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 - - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor - - MONGODB_REPLICA_SET_KEY=vector + - MONGODB_ADVERTISED_HOSTNAME=mongodb-secondary + - MONGODB_REPLICA_SET_MODE=secondary + - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary + - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 + - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor + - MONGODB_REPLICA_SET_KEY=vector mongodb-arbiter: image: docker.io/bitnamilegacy/mongodb:${CONFIG_VERSION} depends_on: - - mongodb-primary + - mongodb-primary environment: - - MONGODB_ADVERTISED_HOSTNAME=mongodb-arbiter - - MONGODB_REPLICA_SET_MODE=arbiter - - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary - - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 - - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor - - MONGODB_REPLICA_SET_KEY=vector + - MONGODB_ADVERTISED_HOSTNAME=mongodb-arbiter + - MONGODB_REPLICA_SET_MODE=arbiter + - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary + - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 + - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor + - MONGODB_REPLICA_SET_KEY=vector networks: default: diff --git a/tests/integration/mongodb/config/test.yaml b/tests/integration/mongodb/config/test.yaml index 10da02aca38dc..3668030a20fb5 100644 --- a/tests/integration/mongodb/config/test.yaml +++ b/tests/integration/mongodb/config/test.yaml @@ -1,7 +1,7 @@ features: -- mongodb_metrics-integration-tests + - mongodb_metrics-integration-tests -test_filter: '::mongodb_metrics::' +test_filter: "::mongodb_metrics::" env: PRIMARY_MONGODB_ADDRESS: mongodb://root:toor@mongodb-primary @@ -13,7 +13,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/mongodb_metrics.rs" -- "src/sources/mongodb_metrics/**" -- "src/sources/util/**" -- "tests/integration/mongodb/**" + - "src/internal_events/mongodb_metrics.rs" + - "src/sources/mongodb_metrics/**" + - "src/sources/util/**" + - "tests/integration/mongodb/**" diff --git a/tests/integration/mqtt/config/compose.yaml b/tests/integration/mqtt/config/compose.yaml index cc238cadf050c..9ce016da98cb1 100644 --- a/tests/integration/mqtt/config/compose.yaml +++ b/tests/integration/mqtt/config/compose.yaml @@ -1,10 +1,10 @@ -version: '3' +version: "3" services: emqx: image: docker.io/emqx:${CONFIG_VERSION} ports: - - 1883:1883 + - 1883:1883 networks: default: diff --git a/tests/integration/mqtt/config/test.yaml b/tests/integration/mqtt/config/test.yaml index ec0c637e094d1..1119ca931faf4 100644 --- a/tests/integration/mqtt/config/test.yaml +++ b/tests/integration/mqtt/config/test.yaml @@ -1,13 +1,13 @@ features: -- mqtt-integration-tests + - mqtt-integration-tests -test_filter: '::mqtt::' +test_filter: "::mqtt::" matrix: - version: ['5.0.15'] + version: ["5.0.15"] paths: -- "src/internal_events/mqtt.rs" -- "src/sinks/mqtt/**" -- "src/sinks/util/**" -- "src/sources/mqtt/**" + - "src/internal_events/mqtt.rs" + - "src/sinks/mqtt/**" + - "src/sinks/util/**" + - "src/sources/mqtt/**" diff --git a/tests/integration/nats/config/compose.yaml b/tests/integration/nats/config/compose.yaml index f77020b57afee..a556ffd61f351 100644 --- a/tests/integration/nats/config/compose.yaml +++ b/tests/integration/nats/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: nats: @@ -6,50 +6,50 @@ services: nats-userpass: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --user - - natsuser - - --pass - - natspass + - --user + - natsuser + - --pass + - natspass nats-token: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --auth - - secret + - --auth + - secret nats-nkey: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-nkey.conf + - --config + - /usr/share/nats/config/nats-nkey.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-tls: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-tls.conf + - --config + - /usr/share/nats/config/nats-tls.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-tls-client-cert: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-tls-client-cert.conf + - --config + - /usr/share/nats/config/nats-tls-client-cert.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-jwt: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-jwt.conf + - --config + - /usr/share/nats/config/nats-jwt.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-jetstream-test: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-jetstream.conf + - --config + - /usr/share/nats/config/nats-jetstream.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config networks: default: diff --git a/tests/integration/nginx/config/compose.yaml b/tests/integration/nginx/config/compose.yaml index 3caf63a002f11..4b0f50d0d92ea 100644 --- a/tests/integration/nginx/config/compose.yaml +++ b/tests/integration/nginx/config/compose.yaml @@ -1,25 +1,25 @@ -version: '3' +version: "3" services: squid: image: docker.io/ubuntu/squid:latest depends_on: - - nginx-proxy + - nginx-proxy networks: - - default - - proxy + - default + - proxy nginx: image: docker.io/nginx:${CONFIG_VERSION} volumes: - - ../data/:/etc/nginx:ro + - ../data/:/etc/nginx:ro networks: - - default + - default nginx-proxy: image: docker.io/nginx:${CONFIG_VERSION} volumes: - - ../data/:/etc/nginx:ro + - ../data/:/etc/nginx:ro networks: - - proxy + - proxy networks: default: diff --git a/tests/integration/nginx/config/test.yaml b/tests/integration/nginx/config/test.yaml index 8d9014b657787..e23372f596139 100644 --- a/tests/integration/nginx/config/test.yaml +++ b/tests/integration/nginx/config/test.yaml @@ -1,7 +1,7 @@ features: -- nginx-integration-tests + - nginx-integration-tests -test_filter: '::nginx_metrics::' +test_filter: "::nginx_metrics::" runner: env: @@ -15,7 +15,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/nginx_metrics.rs" -- "src/sources/nginx_metrics/**" -- "src/sources/util/**" -- "tests/integration/nginx/**" + - "src/internal_events/nginx_metrics.rs" + - "src/sources/nginx_metrics/**" + - "src/sources/util/**" + - "tests/integration/nginx/**" diff --git a/tests/integration/opentelemetry/config/compose.yaml b/tests/integration/opentelemetry/config/compose.yaml index d8f0c95d20fbc..f65495b9bc2ae 100644 --- a/tests/integration/opentelemetry/config/compose.yaml +++ b/tests/integration/opentelemetry/config/compose.yaml @@ -1,10 +1,10 @@ -version: '3' +version: "3" services: opentelemetry-collector: image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION} volumes: - - ../data/config.yaml:/etc/otelcol-contrib/config.yaml + - ../data/config.yaml:/etc/otelcol-contrib/config.yaml networks: default: diff --git a/tests/integration/opentelemetry/config/test.yaml b/tests/integration/opentelemetry/config/test.yaml index 91a2968fdfae0..0a3af636f56b5 100644 --- a/tests/integration/opentelemetry/config/test.yaml +++ b/tests/integration/opentelemetry/config/test.yaml @@ -1,7 +1,7 @@ features: -- opentelemetry-integration-tests + - opentelemetry-integration-tests -test_filter: '::opentelemetry::' +test_filter: "::opentelemetry::" runner: env: @@ -14,6 +14,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/opentelemetry/**" -- "src/sources/util/**" -- "tests/integration/opentelemetry/**" + - "src/sources/opentelemetry/**" + - "src/sources/util/**" + - "tests/integration/opentelemetry/**" diff --git a/tests/integration/opentelemetry/data/config.yaml b/tests/integration/opentelemetry/data/config.yaml index 66c8fe34e6112..ee0fe7fa36126 100644 --- a/tests/integration/opentelemetry/data/config.yaml +++ b/tests/integration/opentelemetry/data/config.yaml @@ -25,10 +25,10 @@ service: pipelines: logs: receivers: [otlp] - exporters: [otlp,otlphttp] + exporters: [otlp, otlphttp] traces: receivers: [otlp] - exporters: [otlp,otlphttp] + exporters: [otlp, otlphttp] metrics: receivers: [otlp] - exporters: [otlp,otlphttp] + exporters: [otlp, otlphttp] diff --git a/tests/integration/postgres/config/compose.yaml b/tests/integration/postgres/config/compose.yaml index 017e497c81811..40970c5f3da28 100644 --- a/tests/integration/postgres/config/compose.yaml +++ b/tests/integration/postgres/config/compose.yaml @@ -1,16 +1,16 @@ -version: '3' +version: "3" services: postgres: image: docker.io/postgres:${CONFIG_VERSION} command: /postgres-init.sh environment: - - POSTGRES_USER=vector - - POSTGRES_PASSWORD=vector + - POSTGRES_USER=vector + - POSTGRES_PASSWORD=vector volumes: - - socket:/var/run/postgresql - - ../data/postgres-init.sh:/postgres-init.sh:ro - - ../../../data/ca:/certs:ro + - socket:/var/run/postgresql + - ../data/postgres-init.sh:/postgres-init.sh:ro + - ../../../data/ca:/certs:ro volumes: # Use external volume 'postgres_socket' that's shared with the test runner diff --git a/tests/integration/postgres/config/test.yaml b/tests/integration/postgres/config/test.yaml index 85c012a11fa5f..905d4f53ccda5 100644 --- a/tests/integration/postgres/config/test.yaml +++ b/tests/integration/postgres/config/test.yaml @@ -1,6 +1,6 @@ features: -- postgresql_metrics-integration-tests -- postgres_sink-integration-tests + - postgresql_metrics-integration-tests + - postgres_sink-integration-tests test_filter: ::postgres @@ -13,13 +13,13 @@ runner: postgres_socket: /pg_socket matrix: - version: ['13.1'] + version: ["13.1"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/postgresql_metrics.rs" -- "src/sinks/postgres/**" -- "src/sources/postgresql_metrics.rs" -- "src/sources/util/**" -- "tests/integration/postgres/**" + - "src/internal_events/postgresql_metrics.rs" + - "src/sinks/postgres/**" + - "src/sources/postgresql_metrics.rs" + - "src/sources/util/**" + - "tests/integration/postgres/**" diff --git a/tests/integration/prometheus/config/compose.yaml b/tests/integration/prometheus/config/compose.yaml index a945f730ce834..3ff987c6d5189 100644 --- a/tests/integration/prometheus/config/compose.yaml +++ b/tests/integration/prometheus/config/compose.yaml @@ -1,26 +1,26 @@ -version: '3' +version: "3" services: influxdb-v1: image: docker.io/influxdb:${CONFIG_INFLUXDB} environment: - - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_REPORTING_DISABLED=true influxdb-v1-tls: image: docker.io/influxdb:${CONFIG_INFLUXDB} environment: - - INFLUXDB_REPORTING_DISABLED=true - - INFLUXDB_HTTP_HTTPS_ENABLED=true - - INFLUXDB_HTTP_BIND_ADDRESS=:8087 - - INFLUXDB_BIND_ADDRESS=:8089 - - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem - - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem + - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_HTTP_HTTPS_ENABLED=true + - INFLUXDB_HTTP_BIND_ADDRESS=:8087 + - INFLUXDB_BIND_ADDRESS=:8089 + - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem + - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem volumes: - - ../../../data/ca:/etc/ssl:ro + - ../../../data/ca:/etc/ssl:ro prometheus: image: docker.io/prom/prometheus:${CONFIG_PROMETHEUS} command: --config.file=/etc/vector/prometheus.yaml volumes: - - ../data/prometheus.yaml:/etc/vector/prometheus.yaml:ro + - ../data/prometheus.yaml:/etc/vector/prometheus.yaml:ro networks: default: diff --git a/tests/integration/prometheus/config/test.yaml b/tests/integration/prometheus/config/test.yaml index 455338053e2ff..ac79ea17c3d8e 100644 --- a/tests/integration/prometheus/config/test.yaml +++ b/tests/integration/prometheus/config/test.yaml @@ -1,21 +1,21 @@ -test_filter: '::prometheus::remote_write::' +test_filter: "::prometheus::remote_write::" features: -- prometheus-integration-tests + - prometheus-integration-tests env: REMOTE_WRITE_SOURCE_RECEIVE_ADDRESS: runner:9102 matrix: - prometheus: ['v2.33.4'] - influxdb: ['1.8'] + prometheus: ["v2.33.4"] + influxdb: ["1.8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/prometheus.rs" -- "src/sources/prometheus/**" -- "src/sources/util/**" -- "src/sinks/prometheus/**" -- "src/sinks/util/**" -- "tests/integration/prometheus/**" + - "src/internal_events/prometheus.rs" + - "src/sources/prometheus/**" + - "src/sources/util/**" + - "src/sinks/prometheus/**" + - "src/sinks/util/**" + - "tests/integration/prometheus/**" diff --git a/tests/integration/prometheus/data/prometheus.yaml b/tests/integration/prometheus/data/prometheus.yaml index 5e9a6ac09a312..79ce0d180acaa 100644 --- a/tests/integration/prometheus/data/prometheus.yaml +++ b/tests/integration/prometheus/data/prometheus.yaml @@ -3,12 +3,12 @@ global: scrape_configs: # sinks::prometheus::exporter::integration_tests - - job_name: 'integration-test-1' + - job_name: "integration-test-1" static_configs: - - targets: ['127.0.0.1:9101'] + - targets: ["127.0.0.1:9101"] remote_write: # sources::prometheus::remote_write::integration_tests - - url: 'http://runner:9102' + - url: "http://runner:9102" remote_timeout: 1s - name: 'integration-test-2' + name: "integration-test-2" diff --git a/tests/integration/pulsar/config/compose.yaml b/tests/integration/pulsar/config/compose.yaml index cca9c9de8d713..5d6c4e86249f9 100644 --- a/tests/integration/pulsar/config/compose.yaml +++ b/tests/integration/pulsar/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: pulsar: diff --git a/tests/integration/pulsar/config/test.yaml b/tests/integration/pulsar/config/test.yaml index a273fb4e9f163..51dbb1100c9da 100644 --- a/tests/integration/pulsar/config/test.yaml +++ b/tests/integration/pulsar/config/test.yaml @@ -1,7 +1,7 @@ features: -- pulsar-integration-tests + - pulsar-integration-tests -test_filter: '::pulsar::integration_tests::' +test_filter: "::pulsar::integration_tests::" env: PULSAR_HOST: pulsar @@ -14,7 +14,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/pulsar.rs" -- "src/sinks/pulsar/**" -- "src/sinks/util/**" -- "tests/integration/pulsar/**" + - "src/internal_events/pulsar.rs" + - "src/sinks/pulsar/**" + - "src/sinks/util/**" + - "tests/integration/pulsar/**" diff --git a/tests/integration/redis/config/compose.yaml b/tests/integration/redis/config/compose.yaml index 4f0a909a6e455..845c99eff0a62 100644 --- a/tests/integration/redis/config/compose.yaml +++ b/tests/integration/redis/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: redis-primary: diff --git a/tests/integration/redis/config/test.yaml b/tests/integration/redis/config/test.yaml index fdfc54e0446ec..6129d66a052fb 100644 --- a/tests/integration/redis/config/test.yaml +++ b/tests/integration/redis/config/test.yaml @@ -1,5 +1,5 @@ features: -- redis-integration-tests + - redis-integration-tests test_filter: "::redis::" @@ -13,9 +13,9 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/redis.rs" -- "src/sources/redis/**" -- "src/sources/util/**" -- "src/sinks/redis.rs" -- "src/sinks/util/**" -- "tests/integration/redis/**" + - "src/internal_events/redis.rs" + - "src/sources/redis/**" + - "src/sources/util/**" + - "src/sinks/redis.rs" + - "src/sinks/util/**" + - "tests/integration/redis/**" diff --git a/tests/integration/shutdown/config/compose.yaml b/tests/integration/shutdown/config/compose.yaml index 81a35dd64ec94..66c16b7fc5f3a 100644 --- a/tests/integration/shutdown/config/compose.yaml +++ b/tests/integration/shutdown/config/compose.yaml @@ -1,39 +1,39 @@ -version: '3' +version: "3" services: zookeeper: image: docker.io/confluentinc/cp-zookeeper:${CONFIG_VERSION} ports: - - 2181:2181 + - 2181:2181 environment: - - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_CLIENT_PORT=2181 kafka: image: docker.io/confluentinc/cp-kafka:7.6.1 depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - ZOOKEEPER_SASL_ENABLED=false - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 - - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 - - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 - - KAFKA_SSL_CLIENT_AUTH=none - - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf - - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ZOOKEEPER_SASL_ENABLED=false + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 + - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 + - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 + - KAFKA_SSL_CLIENT_AUTH=none + - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf + - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN ports: - - 9091:9091 - - 9092:9092 - - 9093:9093 + - 9091:9091 + - 9092:9092 + - 9093:9093 volumes: - - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro - - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro - - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf + - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro + - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro + - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf networks: default: diff --git a/tests/integration/shutdown/config/test.yaml b/tests/integration/shutdown/config/test.yaml index c32f2e010940a..0086db0d6f175 100644 --- a/tests/integration/shutdown/config/test.yaml +++ b/tests/integration/shutdown/config/test.yaml @@ -1,9 +1,9 @@ args: -- --test-threads -- '4' + - --test-threads + - "4" features: -- shutdown-tests + - shutdown-tests test: "integration" diff --git a/tests/integration/splunk/config/compose.yaml b/tests/integration/splunk/config/compose.yaml index a252674d2bffa..4cff3cab1ca0d 100644 --- a/tests/integration/splunk/config/compose.yaml +++ b/tests/integration/splunk/config/compose.yaml @@ -1,18 +1,18 @@ -version: '3' +version: "3" services: splunk-hec: image: docker.io/splunk/splunk:${CONFIG_VERSION} environment: - - SPLUNK_START_ARGS=--accept-license - - SPLUNK_PASSWORD=password - - SPLUNK_HEC_TOKEN=abcd1234 + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_PASSWORD=password + - SPLUNK_HEC_TOKEN=abcd1234 volumes: - - ../data/splunk/default.yml:/tmp/defaults/default.yml + - ../data/splunk/default.yml:/tmp/defaults/default.yml ports: - - 8000:8000 - - 8088:8088 - - 8089:8089 + - 8000:8000 + - 8088:8088 + - 8089:8089 networks: default: diff --git a/tests/integration/splunk/config/test.yaml b/tests/integration/splunk/config/test.yaml index 71a5c7994fe4a..2e745fab67352 100644 --- a/tests/integration/splunk/config/test.yaml +++ b/tests/integration/splunk/config/test.yaml @@ -1,7 +1,7 @@ features: -- splunk-integration-tests + - splunk-integration-tests -test_filter: '::splunk_hec::' +test_filter: "::splunk_hec::" env: SPLUNK_API_ADDRESS: https://splunk-hec:8089 @@ -13,9 +13,9 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/splunk_hec.rs" -- "src/sources/splunk_hec/**" -- "src/sources/util/**" -- "src/sinks/splunk_hec/**" -- "src/sinks/util/**" -- "tests/integration/splunk/**" + - "src/internal_events/splunk_hec.rs" + - "src/sources/splunk_hec/**" + - "src/sources/util/**" + - "src/sinks/splunk_hec/**" + - "src/sinks/util/**" + - "tests/integration/splunk/**" diff --git a/tests/integration/webhdfs/config/compose.yaml b/tests/integration/webhdfs/config/compose.yaml index f3b5a51519513..17fd3d9eb69b5 100644 --- a/tests/integration/webhdfs/config/compose.yaml +++ b/tests/integration/webhdfs/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: namenode: diff --git a/tests/integration/webhdfs/config/test.yaml b/tests/integration/webhdfs/config/test.yaml index f1bd9f88b2ff9..029f12a587b76 100644 --- a/tests/integration/webhdfs/config/test.yaml +++ b/tests/integration/webhdfs/config/test.yaml @@ -1,17 +1,17 @@ -test_filter: '::webhdfs::' +test_filter: "::webhdfs::" features: -- webhdfs-integration-tests + - webhdfs-integration-tests env: WEBHDFS_ENDPOINT: http://namenode.local:9870 matrix: - hadoop: ['2.0.0-hadoop3.2.1-java8'] + hadoop: ["2.0.0-hadoop3.2.1-java8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/webhdfs/**" -- "src/sinks/util/**" -- "tests/integration/webhdfs/**" + - "src/sinks/webhdfs/**" + - "src/sinks/util/**" + - "tests/integration/webhdfs/**" diff --git a/tests/integration/windows-event-log/config/test.yaml b/tests/integration/windows-event-log/config/test.yaml index ed0555b2e6695..6c3c619c54111 100644 --- a/tests/integration/windows-event-log/config/test.yaml +++ b/tests/integration/windows-event-log/config/test.yaml @@ -1,7 +1,7 @@ features: -- windows-event-log-integration-tests + - windows-event-log-integration-tests -test_filter: '::windows_event_log::integration_tests::' +test_filter: "::windows_event_log::integration_tests::" runner: env: {} @@ -12,6 +12,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/windows_event_log/**" -- "src/internal_events/windows_event_log.rs" -- "tests/integration/windows-event-log/**" + - "src/sources/windows_event_log/**" + - "src/internal_events/windows_event_log.rs" + - "tests/integration/windows-event-log/**" diff --git a/tests/namespacing/ignore-invalid/sinks/es_cluster.json b/tests/namespacing/ignore-invalid/sinks/es_cluster.json index 6dd5e392cf253..c72ff984f709a 100644 --- a/tests/namespacing/ignore-invalid/sinks/es_cluster.json +++ b/tests/namespacing/ignore-invalid/sinks/es_cluster.json @@ -1,8 +1,6 @@ { - "inputs": [ - "apache_sample" - ], + "inputs": ["apache_sample"], "type": "elasticsearch", "endpoint": "http://79.12.221.222:9200", - "bulk": {"index": "vector-%Y-%m-%d"} + "bulk": { "index": "vector-%Y-%m-%d" } } diff --git a/tests/namespacing/success/sinks/es_cluster.json b/tests/namespacing/success/sinks/es_cluster.json index 6dd5e392cf253..c72ff984f709a 100644 --- a/tests/namespacing/success/sinks/es_cluster.json +++ b/tests/namespacing/success/sinks/es_cluster.json @@ -1,8 +1,6 @@ { - "inputs": [ - "apache_sample" - ], + "inputs": ["apache_sample"], "type": "elasticsearch", "endpoint": "http://79.12.221.222:9200", - "bulk": {"index": "vector-%Y-%m-%d"} + "bulk": { "index": "vector-%Y-%m-%d" } } diff --git a/vdev/src/commands/check/fmt.rs b/vdev/src/commands/check/fmt.rs index b01f6622218a2..1547d7b015808 100644 --- a/vdev/src/commands/check/fmt.rs +++ b/vdev/src/commands/check/fmt.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use crate::app; +use crate::{app, commands::fmt::PRETTIER_EXTENSIONS, utils::git::git_ls_files}; /// Check that all files are formatted properly #[derive(clap::Args, Debug)] @@ -13,6 +13,21 @@ impl Cli { app::exec("scripts/check-style.sh", ["--all"], true)?; info!("Checking Rust formatting..."); - app::exec("cargo", ["fmt", "--", "--check"], true) + app::exec("cargo", ["fmt", "--", "--check"], true)?; + + for ext in PRETTIER_EXTENSIONS { + let files = git_ls_files(Some(ext))?; + if files.is_empty() { + continue; + } + info!("Checking prettier formatting for {ext} files..."); + let args: Vec<&str> = ["--ignore-path", ".prettierignore", "--check"] + .into_iter() + .chain(files.iter().map(String::as_str)) + .collect(); + app::exec("prettier", &args, true)?; + } + + Ok(()) } } diff --git a/vdev/src/commands/fmt.rs b/vdev/src/commands/fmt.rs index d53c6758ca09d..f1f7353b970d6 100644 --- a/vdev/src/commands/fmt.rs +++ b/vdev/src/commands/fmt.rs @@ -1,6 +1,9 @@ use anyhow::Result; -use crate::app; +use crate::{app, utils::git::git_ls_files}; + +pub(crate) const PRETTIER_EXTENSIONS: &[&str] = + &["*.yml", "*.yaml", "*.js", "*.ts", "*.tsx", "*.json"]; /// Apply format changes across the repository #[derive(clap::Args, Debug)] @@ -13,6 +16,21 @@ impl Cli { app::exec("scripts/check-style.sh", ["--fix"], true)?; info!("Formatting Rust code..."); - app::exec("cargo", ["fmt", "--all"], true) + app::exec("cargo", ["fmt", "--all"], true)?; + + for ext in PRETTIER_EXTENSIONS { + let files = git_ls_files(Some(ext))?; + if files.is_empty() { + continue; + } + info!("Formatting {ext} files with prettier..."); + let args: Vec<&str> = ["--ignore-path", ".prettierignore", "--write"] + .into_iter() + .chain(files.iter().map(String::as_str)) + .collect(); + app::exec("prettier", &args, true)?; + } + + Ok(()) } } diff --git a/website/.htmltest.external.yml b/website/.htmltest.external.yml index 3508294079c6f..3df01b8f3309d 100644 --- a/website/.htmltest.external.yml +++ b/website/.htmltest.external.yml @@ -4,11 +4,11 @@ CheckExternal: true IgnoreAltMissing: true IgnoreEmptyHref: true IgnoreURLs: -# We have to ignore this due to rate limiting -- "github.com" -- "packages.timber.io" -- "localhost" -# Somehow this link allows shows up as broken, so we'll exclude it -- "https://www.microsoft.com/en-us/windows" -# Somehow these links always come up as 5xx even though they work fine in the browser -- "maxmind.com" + # We have to ignore this due to rate limiting + - "github.com" + - "packages.timber.io" + - "localhost" + # Somehow this link allows shows up as broken, so we'll exclude it + - "https://www.microsoft.com/en-us/windows" + # Somehow these links always come up as 5xx even though they work fine in the browser + - "maxmind.com" diff --git a/website/.prettierrc.json b/website/.prettierrc.json deleted file mode 100644 index 7b17711cfff7a..0000000000000 --- a/website/.prettierrc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "plugins": ["prettier-plugin-go-template"], - "overrides": [ - { - "files": ["*.html"], - "options": { - "parser": "go-template" - } - } - ], - "tabWidth": 2, - "useTabs": false, - "semi": false, - "singleQuote": true, - "trailingComma": "none", - "printWidth": 120 -} diff --git a/website/algolia.json b/website/algolia.json index d0c7622b530bf..1a88269b7d9e4 100644 --- a/website/algolia.json +++ b/website/algolia.json @@ -3,47 +3,24 @@ "minWordSizefor2Typos": 8, "hitsPerPage": 10, "maxValuesPerFacet": 100, - "searchableAttributes": [ - "title", - "content", - "unordered(hierarchy)", - "unordered(tags)" - ], + "searchableAttributes": ["title", "content", "unordered(hierarchy)", "unordered(tags)"], "numericAttributesToIndex": null, "attributesToRetrieve": null, "unretrievableAttributes": null, "optionalWords": null, "attributesForFaceting": null, - "attributesToSnippet": [ - "title:10", - "content:10" - ], + "attributesToSnippet": ["title:10", "content:10"], "attributesToHighlight": null, "paginationLimitedTo": 1000, "attributeForDistinct": null, "exactOnSingleWordQuery": "attribute", - "ranking": [ - "typo", - "geo", - "words", - "filters", - "proximity", - "attribute", - "exact", - "custom" - ], - "customRanking": [ - "desc(level)", - "desc(ranking)" - ], + "ranking": ["typo", "geo", "words", "filters", "proximity", "attribute", "exact", "custom"], + "customRanking": ["desc(level)", "desc(ranking)"], "separatorsToIndex": "", "removeWordsIfNoResults": "none", "queryType": "prefixLast", "highlightPreTag": "", "highlightPostTag": "", "snippetEllipsisText": "…", - "alternativesAsExact": [ - "ignorePlurals", - "singleWordSynonym" - ] + "alternativesAsExact": ["ignorePlurals", "singleWordSynonym"] } diff --git a/website/assets/js/below.js b/website/assets/js/below.js index f61004ef2a746..0123d5eb2c36c 100644 --- a/website/assets/js/below.js +++ b/website/assets/js/below.js @@ -1,16 +1,16 @@ -import tocbot from 'tocbot'; +import tocbot from "tocbot"; // Table of contents for documentation pages const tableOfContents = () => { - if (document.getElementById('toc')) { + if (document.getElementById("toc")) { tocbot.init({ - tocSelector: '#toc', - contentSelector: '#page-content', - headingSelector: 'h2,h3,h4,h5', + tocSelector: "#toc", + contentSelector: "#page-content", + headingSelector: "h2,h3,h4,h5", scrollSmoothDuration: 400 }); } -} +}; const showCodeFilename = () => { var els = document.getElementsByClassName("highlight"); @@ -21,9 +21,9 @@ const showCodeFilename = () => { els[i].parentNode.insertBefore(newNode, els[i]); } } -} +}; -document.addEventListener('DOMContentLoaded', () => { +document.addEventListener("DOMContentLoaded", () => { // search.start(); tableOfContents(); diff --git a/website/assets/js/home.tsx b/website/assets/js/home.tsx index fb92927ac4fe8..e6271eb538d16 100644 --- a/website/assets/js/home.tsx +++ b/website/assets/js/home.tsx @@ -28,15 +28,12 @@ interface IGlobeProps { const markers: GeoJSON.Point[] = markerData.map((m) => ({ type: "Point", - coordinates: [m.coordinates[0], m.coordinates[1]], + coordinates: [m.coordinates[0], m.coordinates[1]] })); const topo: unknown = countryData; const topology = topo as Topology; -const geojson = feature( - topology, - topology.objects.land -) as GeoJSON.FeatureCollection; +const geojson = feature(topology, topology.objects.land) as GeoJSON.FeatureCollection; const countries = geojson.features; export const Globe: React.FC = animated( @@ -66,30 +63,20 @@ export const Globe: React.FC = animated( return (
- + {countries.map((d, i) => ( ))} {markers.map((m, i) => { - const coordinates: [number, number] = [ - m.coordinates[0], - m.coordinates[1], - ]; + const coordinates: [number, number] = [m.coordinates[0], m.coordinates[1]]; const position = projection(coordinates); const x = position ? position[0] : undefined; const y = position ? position[1] : undefined; - const invertedCoordinates = - projection.invert && projection.invert(center); + const invertedCoordinates = projection.invert && projection.invert(center); // hide the marker if rotated out of view - const hideMarker = - geoDistance(coordinates, invertedCoordinates || [0, 0]) > 1.4; + const hideMarker = geoDistance(coordinates, invertedCoordinates || [0, 0]) > 1.4; return ( = animated( } ); -export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ - size = 500, - duration = 50000, -}) => { +export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ size = 500, duration = 50000 }) => { // globe positioning state const [orientation, setOrientation] = React.useState({ lat: 0, long: 0, - degrees: 0, + degrees: 0 }); // set up spring animation @@ -129,8 +113,8 @@ export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ long: orientation.long, rotation: orientation.degrees, config: { - duration, - }, + duration + } }); // globe click handler @@ -146,7 +130,7 @@ export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ setOrientation({ lat: Math.floor(Math.random() * size), long: Math.floor(Math.random() * size), - degrees: orientation.degrees + 360, + degrees: orientation.degrees + 360 }); }, duration); @@ -162,7 +146,7 @@ export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ }; // Main page Vector diagram -function Diagram({className, height, width}) { +function Diagram({ className, height, width }) { const [_, updateState] = useState(); const defaultXPosition = 7; const defaultTextLength = 60; @@ -183,7 +167,12 @@ function Diagram({className, height, width}) { } return ( - + Vector Diagram A lightweight and ultra-fast tool for building observability pipelines @@ -202,118 +191,323 @@ function Diagram({className, height, width}) { - + - - - - + + + + - - - - + + + + - + - + - + - + - + - + - + - + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - + + + + + - + ); -}; +} // Place the components in the DOM const globeRoot = createRoot(document.getElementById("globe")); diff --git a/website/assets/js/search.tsx b/website/assets/js/search.tsx index f322f463ed774..9fb384498f10e 100644 --- a/website/assets/js/search.tsx +++ b/website/assets/js/search.tsx @@ -1,13 +1,13 @@ -import { autocomplete } from '@algolia/autocomplete-js' -import Typesense from 'typesense' -import React, { createElement, Fragment, useEffect, useRef } from 'react' -import { createRoot } from 'react-dom/client' +import { autocomplete } from "@algolia/autocomplete-js"; +import Typesense from "typesense"; +import React, { createElement, Fragment, useEffect, useRef } from "react"; +import { createRoot } from "react-dom/client"; // // Algolia search // const appId = process.env.ALGOLIA_APP_ID -const apiKey = process.env.TYPESENSE_PUBLIC_API_KEY -const indexName = process.env.TYPESENSE_INDEX -const host = process.env.TYPESENSE_HOST +const apiKey = process.env.TYPESENSE_PUBLIC_API_KEY; +const indexName = process.env.TYPESENSE_INDEX; +const host = process.env.TYPESENSE_HOST; // const searchClient = algoliasearch(appId, apiKey) let searchClient = new Typesense.Client({ @@ -15,73 +15,53 @@ let searchClient = new Typesense.Client({ nearestNode: { host: `${host}.a1.typesense.net`, port: 443, - protocol: 'https', + protocol: "https" }, nodes: [ { host: `${host}-1.a1.typesense.net`, port: 443, - protocol: 'https', + protocol: "https" }, { host: `${host}-2.a1.typesense.net`, port: 443, - protocol: 'https', + protocol: "https" }, { host: `${host}-3.a1.typesense.net`, port: 443, - protocol: 'https', - }, + protocol: "https" + } ], - connectionTimeoutSeconds: 2, -}) - + connectionTimeoutSeconds: 2 +}); const CommandIcon: React.FC = ({ children }) => { return ( - + {children} - ) -} + ); +}; const Chevron: React.FC = () => { return ( - - + + - ) -} - + ); +}; const Result = ({ hit, components, category }) => { - const hierarchy = hit.document.hierarchy.concat(hit.document.title) - const isRootPage = hierarchy.length < 1 + const hierarchy = hit.document.hierarchy.concat(hit.document.title); + const isRootPage = hierarchy.length < 1; return ( -
- {category} -
+
{category}
{!isRootPage && @@ -100,34 +80,30 @@ const Result = ({ hit, components, category }) => { {isRootPage && }

- {hit.content && ( - - )} - {!hit.content && ( - {hit.document.itemUrl} - )} + {hit.content && } + {!hit.content && {hit.document.itemUrl}}

- ) -} + ); +}; const Autocomplete = (props) => { - const containerRef = useRef(null) - const panelRootRef = useRef({}) + const containerRef = useRef(null); + const panelRootRef = useRef({}); useEffect(() => { if (!containerRef.current) { - return undefined + return undefined; } const search = autocomplete({ container: containerRef.current, renderer: { createElement, Fragment }, render({ children, state, components }, root) { - const { preview } = state.context as any + const { preview } = state.context as any; if (!panelRootRef.current[root]) { - panelRootRef.current[root] = createRoot(root) + panelRootRef.current[root] = createRoot(root); } panelRootRef.current[root].render( @@ -168,18 +144,18 @@ const Autocomplete = (props) => {
- ) + ); }, - ...props, - }) + ...props + }); return () => { - search.destroy() - } - }, [props]) + search.destroy(); + }; + }, [props]); - return
-} + return
; +}; const Search = () => { return ( @@ -190,64 +166,69 @@ const Search = () => { defaultActiveItemId={0} getSources={({ query }) => [ { - sourceId: 'queryResults', + sourceId: "queryResults", async getItems() { - const results = (query) => searchClient.collections('vector_docs').documents().search({ - q: query, - preset: 'vector_docs_search', - exhaustive_search: true, - highlight_fields: 'content', - highlight_full_fields: 'content' - - }).then((result) => { - // order the hits by page group - // const hits = result.hits.sort((a, b) => (a.document.pageTitle < b.document.pageTitle ? -1 : 1)) - - // add page as category if there are duplicates - const hitsWithCategory = result.hits.map((h, i) => { - const prev = result.hits[i - 1] as any - const title = h.document.pageTitle - - // if no previous hit is in this category - if (!prev) { - return { ...h, category: title } - } - - // skip if there is already one in this category - if (prev && prev.document.pageTitle === title) { - return h - } - - // add category if needed - if (prev && prev.document.pageTitle !== title) { - return { ...h, category: title } - } - - return h - }) - return hitsWithCategory - }) - return await results(query) + const results = (query) => + searchClient + .collections("vector_docs") + .documents() + .search({ + q: query, + preset: "vector_docs_search", + exhaustive_search: true, + highlight_fields: "content", + highlight_full_fields: "content" + }) + .then((result) => { + // order the hits by page group + // const hits = result.hits.sort((a, b) => (a.document.pageTitle < b.document.pageTitle ? -1 : 1)) + + // add page as category if there are duplicates + const hitsWithCategory = result.hits.map((h, i) => { + const prev = result.hits[i - 1] as any; + const title = h.document.pageTitle; + + // if no previous hit is in this category + if (!prev) { + return { ...h, category: title }; + } + + // skip if there is already one in this category + if (prev && prev.document.pageTitle === title) { + return h; + } + + // add category if needed + if (prev && prev.document.pageTitle !== title) { + return { ...h, category: title }; + } + + return h; + }); + return hitsWithCategory; + }); + return await results(query); }, getItemUrl({ item }) { - return item.document.itemUrl + return item.document.itemUrl; }, templates: { item({ item, components }) { - const highlight = item.highlights.length && item.highlights.find(h => h.field === 'content' || {}).value || item.document['content'] - item['content'] = highlight - return + const highlight = + (item.highlights.length && item.highlights.find((h) => h.field === "content" || {}).value) || + item.document["content"]; + item["content"] = highlight; + return ; }, noResults() { - return 'No results found.'; - }, - }, + return "No results found."; + } + } } ]} /> - ) -} - + ); +}; -const searchRoot = createRoot(document.getElementById('site-search')) -searchRoot.render() +const searchRoot = createRoot(document.getElementById("site-search")); +searchRoot.render(); diff --git a/website/babel.config.js b/website/babel.config.js index cfe4e614d29c6..bf0e24707c871 100644 --- a/website/babel.config.js +++ b/website/babel.config.js @@ -1,6 +1,6 @@ -import presetEnv from '@babel/preset-env'; -import presetReact from '@babel/preset-react'; -import presetTypeScript from '@babel/preset-typescript'; +import presetEnv from "@babel/preset-env"; +import presetReact from "@babel/preset-react"; +import presetTypeScript from "@babel/preset-typescript"; export default function (api) { api.cache(true); @@ -9,22 +9,22 @@ export default function (api) { [ presetEnv, { - "useBuiltIns": 'entry', - "corejs": 3 + useBuiltIns: "entry", + corejs: 3 } ], [ presetReact, { - "flow": false, - "typescript": true + flow: false, + typescript: true } ], [ presetTypeScript, { - "isTSX": true, - "allExtensions": true + isTSX: true, + allExtensions: true } ] ]; diff --git a/website/data/redirects.yaml b/website/data/redirects.yaml index 4e7a94f467b56..e8ac227c4248b 100644 --- a/website/data/redirects.yaml +++ b/website/data/redirects.yaml @@ -1,74 +1,74 @@ platforms: -- docker -- kubernetes + - docker + - kubernetes sinks: -- aws_sqs -- aws_cloudwatch_logs -- aws_cloudwatch_metrics -- aws_kinesis_firehose -- aws_kinesis_streams -- aws_s3 -- axiom -- azure_monitor_logs -- clickhouse -- databend -- datadog_logs -- datadog_metrics -- elasticsearch -- file -- gcp_cloud_storage -- gcp_pubsub -- gcp_stackdriver_logs -- gcp_stackdriver_metrics -- honeycomb -- http -- humio_logs -- humio_metrics -- influxdb_logs -- influxdb_metrics -- kafka -- keep -- mezmo -- loki -- nats -- new_relic_logs -- new_relic -- papertrail -- postgres -- prometheus_exporter -- prometheus_remote_write -- pulsar -- sematext_logs -- sematext_metrics -- socket -- splunk_hec -- statsd + - aws_sqs + - aws_cloudwatch_logs + - aws_cloudwatch_metrics + - aws_kinesis_firehose + - aws_kinesis_streams + - aws_s3 + - axiom + - azure_monitor_logs + - clickhouse + - databend + - datadog_logs + - datadog_metrics + - elasticsearch + - file + - gcp_cloud_storage + - gcp_pubsub + - gcp_stackdriver_logs + - gcp_stackdriver_metrics + - honeycomb + - http + - humio_logs + - humio_metrics + - influxdb_logs + - influxdb_metrics + - kafka + - keep + - mezmo + - loki + - nats + - new_relic_logs + - new_relic + - papertrail + - postgres + - prometheus_exporter + - prometheus_remote_write + - pulsar + - sematext_logs + - sematext_metrics + - socket + - splunk_hec + - statsd sources: -- apache_metrics -- aws_ecs_metrics -- aws_kinesis_firehose -- aws_s3 -- datadog_logs -- exec -- file -- heroku_logs -- host_metrics -- http -- journald -- kafka -- mongodb_metrics -- nginx_metrics -- postgresql_metrics -- prometheus_remote_write -- prometheus_scrape -- pulsar -- socket -- splunk_hec -- statsd -- stdin -- syslog + - apache_metrics + - aws_ecs_metrics + - aws_kinesis_firehose + - aws_s3 + - datadog_logs + - exec + - file + - heroku_logs + - host_metrics + - http + - journald + - kafka + - mongodb_metrics + - nginx_metrics + - postgresql_metrics + - prometheus_remote_write + - prometheus_scrape + - pulsar + - socket + - splunk_hec + - statsd + - stdin + - syslog # /guides/integrate/platforms/${key}/* /docs/reference/configuration/sources/${source} multi: diff --git a/website/postcss.config.js b/website/postcss.config.js index 96e73589f6611..ccd51f20f80bc 100644 --- a/website/postcss.config.js +++ b/website/postcss.config.js @@ -1,7 +1,7 @@ -import postcssImport from 'postcss-import'; -import tailwindCss from 'tailwindcss'; -import autoprefixer from 'autoprefixer'; -import purgecssPlugin from '@fullhuman/postcss-purgecss'; +import postcssImport from "postcss-import"; +import tailwindCss from "tailwindcss"; +import autoprefixer from "autoprefixer"; +import purgecssPlugin from "@fullhuman/postcss-purgecss"; // These are classes for things that are applied by JS, and thus missed by Hugo. // See assets/js/*.js for places where this happens. @@ -40,12 +40,12 @@ const safeClasses = { "text-md", "text-sm", "w-2", - "w-3", + "w-3" ] }; const purgecss = purgecssPlugin({ - content: ['./hugo_stats.json'], + content: ["./hugo_stats.json"], safelist: safeClasses, defaultExtractor: (content) => { const broadMatches = content.match(/[^<>"'`\s]*[^<>"'`\s:]/g) || []; @@ -59,6 +59,6 @@ export default { postcssImport, tailwindCss, autoprefixer, - ...(process.env.HUGO_ENVIRONMENT === 'production' ? [purgecss] : []) + ...(process.env.HUGO_ENVIRONMENT === "production" ? [purgecss] : []) ] -} +}; diff --git a/website/scripts/create-config-examples.js b/website/scripts/create-config-examples.js index b9096fd4d3b1d..692340b359413 100755 --- a/website/scripts/create-config-examples.js +++ b/website/scripts/create-config-examples.js @@ -1,7 +1,7 @@ -import fs from 'fs'; -import chalk from 'chalk'; -import * as TOML from '@iarna/toml'; -import YAML from 'yaml'; +import fs from "fs"; +import chalk from "chalk"; +import * as TOML from "@iarna/toml"; +import YAML from "yaml"; const cueJsonOutput = "data/docs.json"; @@ -10,56 +10,54 @@ const getExampleValue = (param, deepFilter) => { let value; const getArrayValue = (obj) => { - const enumVal = (obj.enum != null) ? [Object.keys(obj.enum)[0]] : null; + const enumVal = obj.enum != null ? [Object.keys(obj.enum)[0]] : null; - const examplesVal = (obj.examples != null && obj.examples.length > 0) ? [obj.examples[0]] : null; + const examplesVal = obj.examples != null && obj.examples.length > 0 ? [obj.examples[0]] : null; return obj.default || examplesVal || enumVal || null; - } + }; const getValue = (obj) => { - const enumVal = (obj.enum != null) ? Object.keys(obj.enum)[0] : null; + const enumVal = obj.enum != null ? Object.keys(obj.enum)[0] : null; - const examplesVal = (obj.examples != null && obj.examples.length > 0) ? obj.examples[0] : null; + const examplesVal = obj.examples != null && obj.examples.length > 0 ? obj.examples[0] : null; return obj.default || examplesVal || enumVal || null; - } + }; - Object.keys(param.type).forEach(k => { + Object.keys(param.type).forEach((k) => { const p = param.type[k]; - if (['array', 'object'].includes(k)) { + if (["array", "object"].includes(k)) { const topType = k; if (p.items && p.items.type) { const typeInfo = p.items.type; - Object.keys(typeInfo).forEach(k => { - if (['array', 'object'].includes(k)) { + Object.keys(typeInfo).forEach((k) => { + if (["array", "object"].includes(k)) { const subType = k; const options = typeInfo[k].options; var subObj = {}; - Object - .keys(options) - .filter(k => deepFilter(options[k])) - .forEach(k => { - Object.keys(options[k].type).forEach(key => { + Object.keys(options) + .filter((k) => deepFilter(options[k])) + .forEach((k) => { + Object.keys(options[k].type).forEach((key) => { const deepTypeInfo = options[k].type[key]; - if (subType === 'array') { + if (subType === "array") { subObj[k] = getArrayValue(deepTypeInfo); } else { subObj[k] = getValue(deepTypeInfo); } - }); }); value = subObj; } else { - if (topType === 'array') { + if (topType === "array") { value = getArrayValue(typeInfo[k]); } else { value = getValue(typeInfo[k]); @@ -75,15 +73,14 @@ const getExampleValue = (param, deepFilter) => { }); return value; -} +}; Object.makeExampleParams = (params, filter, deepFilter) => { var obj = {}; - Object - .keys(params) - .filter(k => filter(params[k])) - .forEach(k => { + Object.keys(params) + .filter((k) => filter(params[k])) + .forEach((k) => { let value = getExampleValue(params[k], deepFilter); if (value) { obj[k] = value; @@ -91,22 +88,22 @@ Object.makeExampleParams = (params, filter, deepFilter) => { }); return obj; -} +}; // Convert object to TOML string const toToml = (obj) => { return TOML.stringify(obj); -} +}; // Convert object to YAML string const toYaml = (obj) => { return `${YAML.stringify(obj)}`; -} +}; // Convert object to JSON string (indented) const toJson = (obj) => { return JSON.stringify(obj, null, 2); -} +}; // Set the example value for a given config parameter const setExampleValue = (exampleConfig, paramName, param) => { @@ -125,7 +122,7 @@ const setExampleValue = (exampleConfig, paramName, param) => { exampleConfig[paramName] = p.examples[0]; } - if (['array', 'object'].includes(k)) { + if (["array", "object"].includes(k)) { if (p.items) { var obj = {}; @@ -157,7 +154,7 @@ const setExampleValue = (exampleConfig, paramName, param) => { } } }); -} +}; // Assemble the "minimal" params for an example config const makeMinimalParams = (configuration) => { @@ -175,7 +172,7 @@ const makeMinimalParams = (configuration) => { } return minimal; -} +}; // Assemble the "advanced" params for an example config const makeAllParams = (configuration) => { @@ -190,7 +187,7 @@ const makeAllParams = (configuration) => { } return optional; -} +}; // Convert the use case examples (`component.examples`) into multi-format const makeUseCaseExamples = (component) => { @@ -210,31 +207,31 @@ const makeUseCaseExamples = (component) => { exampleConfig = { [kindPlural]: { [keyName]: { - "type": component.type, - inputs: ['my-source-or-transform-id'], + type: component.type, + inputs: ["my-source-or-transform-id"], ...extra } } - } + }; } else { exampleConfig = { [kindPlural]: { [keyName]: { - "type": component.type, + type: component.type, ...extra } } - } + }; } // Strip the "log" or "metric" key in the example output let output; if (example.output) { - if (example.output['log']) { - output = example.output['log']; - } else if (example.output['metric']) { - output = example.output['metric']; + if (example.output["log"]) { + output = example.output["log"]; + } else if (example.output["metric"]) { + output = example.output["metric"]; } else { output = example.output; } @@ -248,11 +245,11 @@ const makeUseCaseExamples = (component) => { configuration: { toml: toToml(exampleConfig), yaml: toYaml(exampleConfig), - json: toJson(exampleConfig), + json: toJson(exampleConfig) }, input: example.input, - output: output, - } + output: output + }; useCases.push(useCase); }); @@ -261,12 +258,12 @@ const makeUseCaseExamples = (component) => { } else { return null; } -} +}; const main = () => { try { const debug = process.env.DEBUG === "true" || false; - const data = fs.readFileSync(cueJsonOutput, 'utf8'); + const data = fs.readFileSync(cueJsonOutput, "utf8"); const docs = JSON.parse(data); const components = docs.components; @@ -285,13 +282,13 @@ const main = () => { const minimalParams = Object.makeExampleParams( configuration, - p => p.required || p.minimal, - p => p.required || p.minimal, + (p) => p.required || p.minimal, + (p) => p.required || p.minimal ); const advancedParams = Object.makeExampleParams( configuration, - _ => true, - p => p.required || p.minimal || p.relevant_when, + (_) => true, + (p) => p.required || p.minimal || p.relevant_when ); const useCaseExamples = makeUseCaseExamples(component); @@ -300,13 +297,13 @@ const main = () => { let minimalExampleConfig, advancedExampleConfig; // Sinks and transforms are treated differently because they need an `inputs` field - if (['sinks', 'transforms'].includes(kind)) { + if (["sinks", "transforms"].includes(kind)) { minimalExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - inputs: ['my-source-or-transform-id'], - ...minimalParams, + type: componentType, + inputs: ["my-source-or-transform-id"], + ...minimalParams } } }; @@ -314,9 +311,9 @@ const main = () => { advancedExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - inputs: ['my-source-or-transform-id'], - ...advancedParams, + type: componentType, + inputs: ["my-source-or-transform-id"], + ...advancedParams } } }; @@ -324,8 +321,8 @@ const main = () => { minimalExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - ...minimalParams, + type: componentType, + ...minimalParams } } }; @@ -333,49 +330,47 @@ const main = () => { advancedExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - ...advancedParams, + type: componentType, + ...advancedParams } } }; } - docs['components'][kind][componentType]['examples'] = useCaseExamples; + docs["components"][kind][componentType]["examples"] = useCaseExamples; - docs['components'][kind][componentType]['example_configs'] = { + docs["components"][kind][componentType]["example_configs"] = { minimal: { toml: toToml(minimalExampleConfig), yaml: toYaml(minimalExampleConfig), - json: toJson(minimalExampleConfig), + json: toJson(minimalExampleConfig) }, advanced: { toml: toToml(advancedExampleConfig), yaml: toYaml(advancedExampleConfig), - json: toJson(advancedExampleConfig), - }, + json: toJson(advancedExampleConfig) + } }; } } - // A debugging statement to make sure things are going basically as planned if (debug) { - console.log(docs['components']['sources']['syslog']['examples']); + console.log(docs["components"]["sources"]["syslog"]["examples"]); } - console.log(chalk.green("Success. Finished generating examples for all components.")); console.log(chalk.blue(`Writing generated examples as JSON to ${cueJsonOutput}...`)); // Write back to the JSON file only when not in debug mode if (!debug) { - fs.writeFileSync(cueJsonOutput, JSON.stringify(docs), 'utf8'); + fs.writeFileSync(cueJsonOutput, JSON.stringify(docs), "utf8"); } console.log(chalk.green(`Success. Finished writing example configs to ${cueJsonOutput}.`)); } catch (err) { console.error(err); } -} +}; main(); diff --git a/website/scripts/typesense-index.ts b/website/scripts/typesense-index.ts index 7b14f07d0fbdd..f800e710d1754 100644 --- a/website/scripts/typesense-index.ts +++ b/website/scripts/typesense-index.ts @@ -54,7 +54,7 @@ const tagHierarchy = { h5: 2, h6: 1, li: 1, - p: 1, + p: 1 }; function getPageUrl(file: string) { @@ -75,7 +75,7 @@ function getItemUrl(file: string, { level, domId }: Payload[0]) { // Hash the file name to create a unique ID for the record using the same hash function as the one used in the synapse stream package. function hashString(fileName: string) { - return crypto.createHash('md5').update(fileName).digest('hex') + return crypto.createHash("md5").update(fileName).digest("hex"); } async function indexHTMLFiles( @@ -92,8 +92,8 @@ async function indexHTMLFiles( const $ = cheerio.load(html); const containers = $("#page-content"); const pageTitle = $("meta[name='algolia:title']").attr("content") || ""; - const pageTagsString = $("meta[name='keywords']").attr('content') || ""; - const pageTags: string[] = (pageTagsString === "") ? [] : pageTagsString.split(","); + const pageTagsString = $("meta[name='keywords']").attr("content") || ""; + const pageTags: string[] = pageTagsString === "" ? [] : pageTagsString.split(","); // @ts-ignore $(".algolia-no-index").each((_, d) => $(d).remove()); @@ -114,7 +114,7 @@ async function indexHTMLFiles( tagName: node.tagName, content: $(node) .text() - .replace(/[\n\t]/g, " "), + .replace(/[\n\t]/g, " ") }); } @@ -146,9 +146,10 @@ async function indexHTMLFiles( ranking, hierarchy: [], tags: pageTags, - content: "", + content: "" }; - } else if (item.level === 1) { // h1 logic + } else if (item.level === 1) { + // h1 logic activeRecord.content += item.content; } else if (item.level < activeRecord.level) { algoliaRecords.push({ ...activeRecord }); @@ -164,9 +165,10 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy, activeRecord.title.trim()], tags: pageTags, - content: "", + content: "" }; - } else { // h2-h6 logic + } else { + // h2-h6 logic algoliaRecords.push({ ...activeRecord }); const hierarchySize = activeRecord.hierarchy.length; @@ -184,12 +186,12 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy.slice(0, lastIndex)], tags: pageTags, - content: "", + content: "" }; } if (activeRecord) { - algoliaRecords.push({ ...activeRecord }) + algoliaRecords.push({ ...activeRecord }); } for (const rec of algoliaRecords) { @@ -211,9 +213,7 @@ async function indexHTMLFiles( } } - console.log( - chalk.green(`Success. Updated records for ${files.length} file(s).`) - ); + console.log(chalk.green(`Success. Updated records for ${files.length} file(s).`)); records.push(...algoliaRecords); } @@ -228,56 +228,56 @@ async function buildIndex() { name: "Docs", path: `${publicPath}/docs/introduction/**/**.html`, displayPath: "docs/introduction", - ranking: 50, + ranking: 50 }, { name: "Docs", path: `${publicPath}/docs/administration/**/**.html`, displayPath: "docs/administration", - ranking: 50, + ranking: 50 }, { name: "Docs", path: `${publicPath}/docs/reference/**/**.html`, displayPath: "docs/reference", - ranking: 50, + ranking: 50 }, { name: "Docs", path: `${publicPath}/docs/setup/**/**.html`, displayPath: "docs/setup", - ranking: 50, + ranking: 50 }, { name: "Advanced guides", path: `${publicPath}/guides/advanced/**/**.html`, displayPath: "guides/advanced", - ranking: 40, + ranking: 40 }, { name: "Level up guides", path: `${publicPath}/guides/level-up/**/**.html`, displayPath: "guides/level-up", - ranking: 40, + ranking: 40 }, { name: "Developer guides", path: `${publicPath}/guides/developer/**/**.html`, displayPath: "guides/developer", - ranking: 40, + ranking: 40 }, { name: "Vector highlights", path: `${publicPath}/highlights/**/**.html`, displayPath: "highlights/", - ranking: 40, + ranking: 40 }, { name: "Upgrade guides", path: `${publicPath}/highlights/*-upgrade-guide/**.html`, displayPath: "highlights/", - ranking: 10, - }, + ranking: 10 + } ]; // Recurse through each section and push the resulting records to `allRecords` @@ -286,7 +286,7 @@ async function buildIndex() { // We shouldn't index upgrade guides if (section.name === "Vector highlights") { - files = files.filter((path) => !path.includes("-upgrade-guide")) + files = files.filter((path) => !path.includes("-upgrade-guide")); } console.log(chalk.blue(`Indexing ${section.displayPath}...`)); diff --git a/website/scripts/typesense-sync.ts b/website/scripts/typesense-sync.ts index 65f867de7b98d..9590be0104fb8 100644 --- a/website/scripts/typesense-sync.ts +++ b/website/scripts/typesense-sync.ts @@ -1,19 +1,18 @@ // Will be running in the CI/CD pipeline -import process from 'node:process'; -import { typesenseSync } from 'typesense-sync'; -const configFilePath = './typesense.config.json'; +import process from "node:process"; +import { typesenseSync } from "typesense-sync"; +const configFilePath = "./typesense.config.json"; const config = { configFilePath, - shouldSaveSettings: true, + shouldSaveSettings: true }; typesenseSync(config) .then(() => { - console.log('Typesense sync completed'); + console.log("Typesense sync completed"); }) .catch((err) => { - console.error('An error occurred', err); + console.error("An error occurred", err); process.exit(1); }); - diff --git a/website/typesense.config.json b/website/typesense.config.json index 89b40e66a4ef8..249a303dc895e 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -54,7 +54,7 @@ "optional": true, "sort": false, "stem": false, - "type": "string", + "type": "string", "stem_dictionary": "", "store": true }, From b59c9a25b899223b6633776eeafe2e67fe418640 Mon Sep 17 00:00:00 2001 From: steveduan-IDME Date: Thu, 2 Apr 2026 11:32:25 -0400 Subject: [PATCH 045/364] chore(ci): Add http_server source to semantic PR scope list (#25078) Co-Authored-By: Claude Code --- .github/workflows/semantic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 79a36a94ec7cf..1e134a9f96007 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -160,6 +160,7 @@ jobs: heroku_logs source host_metrics source http_client source + http_server source http_scrape source internal_logs source internal_metrics source From 1a78c93520ab25377b7162c9aa5756f29492f433 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 2 Apr 2026 13:57:21 -0400 Subject: [PATCH 046/364] fix(ci): fix Pulsar TLS integration tests with latest image (#25099) * fix(ci): pin Pulsar integration test image to 4.1.3 The `apachepulsar/pulsar:latest` image moved to a version that breaks TLS connectivity, causing `pulsar_happy_tls` and `consumes_event_with_tls` to fail consistently in CI. Pin to 4.1.3 to unblock the merge queue. Re-enabling `latest` is tracked in #25096. Co-Authored-By: Claude Opus 4.6 (1M context) * Add PULSAR_PREFIX_advertisedAddress=pulsar * fix(ci): revert Pulsar image pin to latest * Revert to version: latest * Fix test.yml format * Delete obsolete comment --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.6 (1M context) --- tests/integration/pulsar/config/compose.yaml | 4 ++++ tests/integration/pulsar/config/test.yaml | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/pulsar/config/compose.yaml b/tests/integration/pulsar/config/compose.yaml index 5d6c4e86249f9..58c2b9344fcad 100644 --- a/tests/integration/pulsar/config/compose.yaml +++ b/tests/integration/pulsar/config/compose.yaml @@ -8,6 +8,10 @@ services: - 6650:6650 - 6651:6651 environment: + # Since apache/pulsar#25238, standalone mode advertises the container's FQDN + # (the container ID) instead of localhost. Set explicitly so TLS hostname + # verification matches the certificate's DNS SAN. + - PULSAR_PREFIX_advertisedAddress=pulsar - PULSAR_PREFIX_brokerServicePortTls=6651 - PULSAR_PREFIX_tlsKeyFilePath=/etc/pulsar/certs/pulsar.key.pem - PULSAR_PREFIX_tlsCertificateFilePath=/etc/pulsar/certs/pulsar.cert.pem diff --git a/tests/integration/pulsar/config/test.yaml b/tests/integration/pulsar/config/test.yaml index 51dbb1100c9da..c136bffaaab24 100644 --- a/tests/integration/pulsar/config/test.yaml +++ b/tests/integration/pulsar/config/test.yaml @@ -7,9 +7,7 @@ env: PULSAR_HOST: pulsar matrix: - # TODO: re-enable `latest` once TLS tests pass with newer Pulsar images. - # See https://github.com/vectordotdev/vector/issues/25096 - version: ["4.1.3"] + version: [latest] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From dedb9667511691731c30c05235c319867d96580c Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 11:19:53 -0400 Subject: [PATCH 047/364] fix(ci): pin npm transitive deps via package-lock.json and npm ci (#25115) Replace `npm install -g` (which resolves transitive dependencies live from the npm registry without a lockfile) with `npm ci` against a committed package-lock.json. This pins the entire dependency tree for markdownlint-cli2, prettier, and datadog-ci, closing the supply chain hole exposed by the axios compromise (2026-03-30). Changes: - Add scripts/environment/npm-tools/ with package.json and package-lock.json as the single source of truth for npm tool versions - Replace maybe_install_npm_package with maybe_install_npm_tools using npm ci --prefix for deterministic installs from the lockfile - Use $(npm config get prefix -g)/bin for symlinks instead of hardcoded /usr/local/bin to respect npm configuration - Use sudo for symlinks only when the target dir is not writable - Verify symlink targets point to our node_modules, not just that binaries exist, to prevent stale or system installs from being used - Early return when no npm tool is requested, so hosts without npm are not broken - Separate cargo and npm caches in CI for cleaner invalidation; npm cache keys hash all files in npm-tools/ - Bump datadog-ci to 5.11.0 to resolve critical simple-git RCE (GHSA-r275-fr43-pm7q) Co-authored-by: Claude Opus 4.6 (1M context) --- .github/actions/setup/action.yml | 18 +- scripts/environment/Dockerfile | 1 + .../environment/npm-tools/package-lock.json | 3378 +++++++++++++++++ scripts/environment/npm-tools/package.json | 9 + scripts/environment/prepare.sh | 67 +- 5 files changed, 3449 insertions(+), 24 deletions(-) create mode 100644 scripts/environment/npm-tools/package-lock.json create mode 100644 scripts/environment/npm-tools/package.json diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index fde57a4b33303..6faa185204ced 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -256,8 +256,8 @@ runs: with: skip-cache: ${{ env.VDEV_NEEDS_COMPILE == 'true' }} - - name: Cache prepare.sh binaries - id: cache-prepare-binaries + - name: Cache cargo binaries + id: cache-cargo-binaries uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: | @@ -270,12 +270,16 @@ runs: ~/.cargo/bin/cargo-llvm-cov ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack - /usr/local/bin/markdownlint-cli2 - /usr/local/bin/prettier - /usr/local/bin/datadog-ci - key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} + key: ${{ runner.os }}-cargo-binaries-${{ hashFiles('scripts/environment/*.sh') }} restore-keys: | - ${{ runner.os }}-prepare-binaries- + ${{ runner.os }}-cargo-binaries- + + - name: Cache npm tools + id: cache-npm-tools + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: scripts/environment/npm-tools/node_modules + key: ${{ runner.os }}-npm-tools-${{ hashFiles('scripts/environment/npm-tools/*') }} - name: Run prepare.sh shell: bash diff --git a/scripts/environment/Dockerfile b/scripts/environment/Dockerfile index fa7d72075dae6..2798a296dd428 100644 --- a/scripts/environment/Dockerfile +++ b/scripts/environment/Dockerfile @@ -16,6 +16,7 @@ WORKDIR /git/vectordotdev/vector # Setup the env COPY scripts/environment/*.sh scripts/environment/ +COPY scripts/environment/npm-tools/ scripts/environment/npm-tools/ RUN ./scripts/environment/bootstrap-ubuntu-24.04.sh # Setup the toolchain diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json new file mode 100644 index 0000000000000..e3d4efa122515 --- /dev/null +++ b/scripts/environment/npm-tools/package-lock.json @@ -0,0 +1,3378 @@ +{ + "name": "environment", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@datadog/datadog-ci": "5.11.0", + "markdownlint-cli2": "0.22.0", + "prettier": "3.8.1" + } + }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@datadog/datadog-ci": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.11.0.tgz", + "integrity": "sha512-4N/WsfnQVYJNxGtDIOhg4sUkZalDo8h+toSXR/GAw6XU81e1bOj3GvleL6IHUQ06KyPdurqPZKB4Za1CbCsT7g==", + "license": "Apache-2.0", + "dependencies": { + "@datadog/datadog-ci-base": "5.11.0", + "@datadog/datadog-ci-plugin-coverage": "5.11.0", + "@datadog/datadog-ci-plugin-deployment": "5.11.0", + "@datadog/datadog-ci-plugin-dora": "5.11.0", + "@datadog/datadog-ci-plugin-gate": "5.11.0", + "@datadog/datadog-ci-plugin-junit": "5.11.0", + "@datadog/datadog-ci-plugin-sarif": "5.11.0", + "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "clipanion": "3.2.1", + "typanion": "3.14.0" + }, + "bin": { + "datadog-ci": "dist/cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@datadog/datadog-ci-base": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.11.0.tgz", + "integrity": "sha512-NACS6zhf2qidgpgrhQH0EJn4x20pwyRAhw75jBRPmjzheaXrpU0438e3kutyNsDGZrw8gHjuRqiccoY3QU7PSQ==", + "license": "Apache-2.0", + "dependencies": { + "@antfu/install-pkg": "1.1.0", + "@types/datadog-metrics": "0.6.1", + "async-retry": "1.3.1", + "axios": "1.13.5", + "chalk": "3.0.0", + "clipanion": "3.2.1", + "datadog-metrics": "0.9.3", + "debug": "4.4.1", + "deep-extend": "0.6.0", + "fast-xml-parser": "5.3.7", + "form-data": "4.0.4", + "glob": "13.0.5", + "inquirer": "8.2.7", + "is-docker": "4.0.0", + "jest-diff": "30.2.0", + "js-yaml": "4.1.1", + "jszip": "3.10.1", + "proxy-agent": "6.4.0", + "semver": "7.6.3", + "simple-git": "3.33.0", + "terminal-link": "2.1.1", + "tiny-async-pool": "2.1.0", + "typanion": "3.14.0", + "upath": "2.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-plugin-aas": "5.11.0", + "@datadog/datadog-ci-plugin-cloud-run": "5.11.0", + "@datadog/datadog-ci-plugin-container-app": "5.11.0", + "@datadog/datadog-ci-plugin-coverage": "5.11.0", + "@datadog/datadog-ci-plugin-deployment": "5.11.0", + "@datadog/datadog-ci-plugin-dora": "5.11.0", + "@datadog/datadog-ci-plugin-gate": "5.11.0", + "@datadog/datadog-ci-plugin-junit": "5.11.0", + "@datadog/datadog-ci-plugin-lambda": "5.11.0", + "@datadog/datadog-ci-plugin-sarif": "5.11.0", + "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "@datadog/datadog-ci-plugin-stepfunctions": "5.11.0", + "@datadog/datadog-ci-plugin-synthetics": "5.11.0", + "@datadog/datadog-ci-plugin-terraform": "5.11.0" + }, + "peerDependenciesMeta": { + "@datadog/datadog-ci-plugin-aas": { + "optional": true + }, + "@datadog/datadog-ci-plugin-cloud-run": { + "optional": true + }, + "@datadog/datadog-ci-plugin-container-app": { + "optional": true + }, + "@datadog/datadog-ci-plugin-coverage": { + "optional": true + }, + "@datadog/datadog-ci-plugin-deployment": { + "optional": true + }, + "@datadog/datadog-ci-plugin-dora": { + "optional": true + }, + "@datadog/datadog-ci-plugin-gate": { + "optional": true + }, + "@datadog/datadog-ci-plugin-junit": { + "optional": true + }, + "@datadog/datadog-ci-plugin-lambda": { + "optional": true + }, + "@datadog/datadog-ci-plugin-sarif": { + "optional": true + }, + "@datadog/datadog-ci-plugin-sbom": { + "optional": true + }, + "@datadog/datadog-ci-plugin-stepfunctions": { + "optional": true + }, + "@datadog/datadog-ci-plugin-synthetics": { + "optional": true + }, + "@datadog/datadog-ci-plugin-terraform": { + "optional": true + } + } + }, + "node_modules/@datadog/datadog-ci-base/node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@datadog/datadog-ci-plugin-coverage": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.11.0.tgz", + "integrity": "sha512-aKzkjbd/NVt+7Sd4QL0qVqUrnyMG0UF63lvNxve7suWQDoIWowAgCPa0ta+jpMYUYCGtjt74H3wL+q0W0G+D+Q==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "3.0.0", + "fast-xml-parser": "5.3.7", + "form-data": "4.0.4", + "upath": "2.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-deployment": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.11.0.tgz", + "integrity": "sha512-FF9L7alDz1XN5L3nYR7xMeZd99U6kxkT+InSyZY21ZHpcuuEngseW1aMgB0iFn3lxZ+1H4z196uCckViTEkjog==", + "license": "Apache-2.0", + "dependencies": { + "axios": "1.13.5", + "chalk": "3.0.0", + "simple-git": "3.33.0" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-dora": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.11.0.tgz", + "integrity": "sha512-DWSyT2tW23XOYCvTEHvD/F9FRIffTN+2wZGYojaD1LM9wJew4xievziBwyD46quCF0OdsbGMLvAyR2hyrrYeZA==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "3.0.0", + "simple-git": "3.33.0" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-gate": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.11.0.tgz", + "integrity": "sha512-WUNr6y292eH+sumAyD5rFhqlU/aJ8vN74h4EJj0SqDpnrDCzMnSDgRyb9GXhv8ZO4GOW6LTEcPmMZhyfYL3b5g==", + "license": "Apache-2.0", + "dependencies": { + "@types/uuid": "9.0.8", + "chalk": "3.0.0", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-junit": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.11.0.tgz", + "integrity": "sha512-I8H09jNR/LDYqiy7TOKv9Gu4+t/o0J6QH6cooKrNboULSDYijVo6w3knANsd2m3Hlcbi1QkTmNpPXIdeGR5Dig==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "3.0.0", + "fast-xml-parser": "5.3.7", + "form-data": "4.0.4", + "upath": "2.0.1", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-sarif": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.11.0.tgz", + "integrity": "sha512-JZuqNzFsW9BZeStLM60f5zdEpG67kh7xqgbeJhqIcloEpDJIEmsJHH487PM0stakcqhhIfzUzvEW744LLQg5Ug==", + "license": "Apache-2.0", + "dependencies": { + "ajv": "8.18.0", + "ajv-formats": "3.0.1", + "chalk": "3.0.0", + "form-data": "4.0.4", + "upath": "2.0.1", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-sbom": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.11.0.tgz", + "integrity": "sha512-lHU65a1Hru7SiwhZtZ9OTd6vyPcRG2U/61imVpvLvhEMSwY2xSLTtZfxWbMi4VJxq81V2zhdifxP5BMJO1FnHw==", + "license": "Apache-2.0", + "dependencies": { + "ajv": "8.18.0", + "ajv-formats": "3.0.1", + "axios": "1.13.5", + "chalk": "3.0.0", + "packageurl-js": "2.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@jest/diff-sequences": { + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", + "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/get-type": { + "version": "30.1.0", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", + "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/schemas": { + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", + "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.34.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@kwsites/file-exists": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", + "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1" + } + }, + "node_modules/@kwsites/promise-deferred": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", + "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", + "license": "MIT" + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.34.49", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", + "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", + "license": "MIT" + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, + "node_modules/@types/datadog-metrics": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@types/datadog-metrics/-/datadog-metrics-0.6.1.tgz", + "integrity": "sha512-p6zVpfmNcXwtcXjgpz7do/fKyfndGhU5sGJVtb5Gn5PvLDiQUAgD0mI/itf/99sBi9DRxeyhFQ9dQF6OxxQNbA==", + "license": "MIT" + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/katex": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", + "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/async-retry": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", + "integrity": "sha512-aiieFW/7h3hY0Bq5d+ktDBejxuwR78vRu9hDUdR8rNhSaQ29VzPL4AoIRG7D/c7tdenwOcKvgPM6tIxB3cB6HA==", + "license": "MIT", + "dependencies": { + "retry": "0.12.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/basic-ftp": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", + "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "license": "ISC", + "engines": { + "node": ">= 10" + } + }, + "node_modules/clipanion": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/clipanion/-/clipanion-3.2.1.tgz", + "integrity": "sha512-dYFdjLb7y1ajfxQopN05mylEpK9ZX0sO1/RfMXdfmwjlIsPkbh4p7A682x++zFPLDCo1x3p82dtljHf5cW2LKA==", + "license": "MIT", + "workspaces": [ + "website" + ], + "dependencies": { + "typanion": "^3.8.0" + }, + "peerDependencies": { + "typanion": "*" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/datadog-metrics": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/datadog-metrics/-/datadog-metrics-0.9.3.tgz", + "integrity": "sha512-BVsBX2t+4yA3tHs7DnB5H01cHVNiGJ/bHA8y6JppJDyXG7s2DLm6JaozPGpgsgVGd42Is1CHRG/yMDQpt877Xg==", + "license": "MIT", + "dependencies": { + "debug": "3.1.0", + "dogapi": "2.8.4" + } + }, + "node_modules/datadog-metrics/node_modules/debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/datadog-metrics/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dogapi": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/dogapi/-/dogapi-2.8.4.tgz", + "integrity": "sha512-065fsvu5dB0o4+ENtLjZILvXMClDNH/yA9H6L8nsdcNiz9l0Hzpn7aQaCOPYXxqyzq4CRPOdwkFXUjDOXfRGbg==", + "license": "MIT", + "dependencies": { + "extend": "^3.0.2", + "json-bigint": "^1.0.0", + "lodash": "^4.17.21", + "minimist": "^1.2.5", + "rc": "^1.2.8" + }, + "bin": { + "dogapi": "bin/dogapi" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-xml-parser": { + "version": "5.3.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.7.tgz", + "integrity": "sha512-JzVLro9NQv92pOM/jTCR6mHlJh2FGwtomH8ZQjhFj/R29P2Fnj38OgPJVtcvYw6SuKClhgYuwUZf5b3rd8u2mA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.2" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", + "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/glob": { + "version": "13.0.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.5.tgz", + "integrity": "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.1", + "minipass": "^7.1.2", + "path-scurry": "^2.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "16.1.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-16.1.1.tgz", + "integrity": "sha512-dW7vl+yiAJSp6aCekaVnVJxurRv7DCOLyXqEG3RYMYUg7AuJ2jCqPkZTA8ooqC2vtnkaMcV5WfFBMuEnTu1OQg==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "fast-glob": "^3.3.3", + "ignore": "^7.0.5", + "is-path-inside": "^4.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/inquirer": { + "version": "8.2.7", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", + "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.0", + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.1", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "figures": "^3.0.0", + "lodash": "^4.17.21", + "mute-stream": "0.0.8", + "ora": "^5.4.1", + "run-async": "^2.4.0", + "rxjs": "^7.5.5", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6", + "wrap-ansi": "^6.0.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/inquirer/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-docker": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-4.0.0.tgz", + "integrity": "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-4.0.0.tgz", + "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/jest-diff": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", + "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", + "license": "MIT", + "dependencies": { + "@jest/diff-sequences": "30.0.1", + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "pretty-format": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-diff/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/jsonc-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/katex": { + "version": "0.16.44", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.44.tgz", + "integrity": "sha512-EkxoDTk8ufHqHlf9QxGwcxeLkWRR3iOuYfRpfORgYfqc8s13bgb+YtRY59NK5ZpRaCwq1kqA6a5lpX8C/eLphQ==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-symbols/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/lru-cache": { + "version": "11.2.7", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", + "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/markdown-it": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdownlint": { + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.40.0.tgz", + "integrity": "sha512-UKybllYNheWac61Ia7T6fzuQNDZimFIpCg2w6hHjgV1Qu0w1TV0LlSgryUGzM0bkKQCBhy2FDhEELB73Kb0kAg==", + "license": "MIT", + "dependencies": { + "micromark": "4.0.2", + "micromark-core-commonmark": "2.0.3", + "micromark-extension-directive": "4.0.0", + "micromark-extension-gfm-autolink-literal": "2.1.0", + "micromark-extension-gfm-footnote": "2.1.0", + "micromark-extension-gfm-table": "2.1.1", + "micromark-extension-math": "3.1.0", + "micromark-util-types": "2.0.2", + "string-width": "8.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + } + }, + "node_modules/markdownlint-cli2": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli2/-/markdownlint-cli2-0.22.0.tgz", + "integrity": "sha512-mOC9BY/XGtdX3M9n3AgERd79F0+S7w18yBBTNIQ453sI87etZfp1z4eajqSMV70CYjbxKe5ktKvT2HCpvcWx9w==", + "license": "MIT", + "dependencies": { + "globby": "16.1.1", + "js-yaml": "4.1.1", + "jsonc-parser": "3.3.1", + "jsonpointer": "5.0.1", + "markdown-it": "14.1.1", + "markdownlint": "0.40.0", + "markdownlint-cli2-formatter-default": "0.0.6", + "micromatch": "4.0.8", + "smol-toml": "1.6.0" + }, + "bin": { + "markdownlint-cli2": "markdownlint-cli2-bin.mjs" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + } + }, + "node_modules/markdownlint-cli2-formatter-default": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/markdownlint-cli2-formatter-default/-/markdownlint-cli2-formatter-default-0.0.6.tgz", + "integrity": "sha512-VVDGKsq9sgzu378swJ0fcHfSicUnMxnL8gnLm/Q4J/xsNJ4e5bA6lvAz7PCzIl0/No0lHyaWdqVD2jotxOSFMQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + }, + "peerDependencies": { + "markdownlint-cli2": ">=0.0.4" + } + }, + "node_modules/markdownlint/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/markdownlint/node_modules/string-width": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", + "integrity": "sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdownlint/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-directive/-/micromark-extension-directive-4.0.0.tgz", + "integrity": "sha512-/C2nqVmXXmiseSSuCdItCMho7ybwwop6RrrRPk0KbOHW21JKoCldC+8rFOaundDoRBUWBnJJcxeA/Kvi34WQXg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-math": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz", + "integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==", + "license": "MIT", + "dependencies": { + "@types/katex": "^0.16.0", + "devlop": "^1.0.0", + "katex": "^0.16.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "license": "ISC" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", + "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/package-manager-detector": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", + "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", + "license": "MIT" + }, + "node_modules/packageurl-js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-2.0.1.tgz", + "integrity": "sha512-N5ixXjzTy4QDQH0Q9YFjqIWd6zH6936Djpl2m9QNFmDv5Fum8q8BjkpAcHNMzOFE0IwQrFhJWex3AN6kS0OSwg==", + "license": "MIT" + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pretty-format": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", + "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.5", + "ansi-styles": "^5.2.0", + "react-is": "^18.3.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/simple-git": { + "version": "3.33.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.33.0.tgz", + "integrity": "sha512-D4V/tGC2sjsoNhoMybKyGoE+v8A60hRawKQ1iFRA1zwuDgGZCBJ4ByOzZ5J8joBbi4Oam0qiPH+GhzmSBwbJng==", + "license": "MIT", + "dependencies": { + "@kwsites/file-exists": "^1.1.1", + "@kwsites/promise-deferred": "^1.1.1", + "debug": "^4.4.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/git-js?sponsor=1" + } + }, + "node_modules/slash": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/smol-toml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", + "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strnum": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.2.tgz", + "integrity": "sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-hyperlinks": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terminal-link": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", + "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^4.2.1", + "supports-hyperlinks": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/tiny-async-pool": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-2.1.0.tgz", + "integrity": "sha512-ltAHPh/9k0STRQqaoUX52NH4ZQYAJz24ZAEwf1Zm+HYg3l9OXTWeqWKyYsHu40wF/F0rxd2N2bk5sLvX2qlSvg==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", + "integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typanion": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/typanion/-/typanion-3.14.0.tgz", + "integrity": "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==", + "license": "MIT", + "workspaces": [ + "website" + ] + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, + "node_modules/unicorn-magic": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.4.0.tgz", + "integrity": "sha512-wH590V9VNgYH9g3lH9wWjTrUoKsjLF6sGLjhR4sH1LWpLmCOH0Zf7PukhDA8BiS7KHe4oPNkcTHqYkj7SOGUOw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/upath": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", + "integrity": "sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==", + "license": "MIT", + "engines": { + "node": ">=4", + "yarn": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json new file mode 100644 index 0000000000000..30d63057b7fed --- /dev/null +++ b/scripts/environment/npm-tools/package.json @@ -0,0 +1,9 @@ +{ + "private": true, + "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", + "dependencies": { + "@datadog/datadog-ci": "5.11.0", + "markdownlint-cli2": "0.22.0", + "prettier": "3.8.1" + } +} diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index e74aba915ad5f..02e100999c8de 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -37,9 +37,8 @@ CARGO_HACK_VERSION="0.6.43" DD_RUST_LICENSE_TOOL_VERSION="1.0.6" CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" -MARKDOWNLINT_CLI2_VERSION="0.22.0" -PRETTIER_VERSION="3.8.1" -DATADOG_CI_VERSION="5.9.0" +# npm tool versions are defined in scripts/environment/npm-tools/package.json +# and pinned (including transitive deps) in npm-tools/package-lock.json. VDEV_VERSION="0.3.0" ALL_MODULES=( @@ -151,22 +150,58 @@ maybe_install_cargo_tool() { fi } -# Helper for NPM packages -# Usage: maybe_install_npm_package -maybe_install_npm_package() { - local tool="$1" - local package="$2" - local version="$3" - local version_pattern="${4:-$version}" - local version_cmd="${5:---version}" # Default to --version, can override with "version" etc. +# Install npm tools from the committed package-lock.json so that every +# transitive dependency version is pinned (no live registry resolution). +# Versions are defined in npm-tools/package.json; npm ci ensures exact lockfile match. +# Note: npm ci installs all packages in the lockfile even if only one tool +# is requested, since it does not support selective installation. +maybe_install_npm_tools() { + local npm_tools=(markdownlint-cli2 prettier datadog-ci) + + # Early return when no npm tool is requested, so hosts without npm + # (e.g. tests/e2e/Dockerfile calling prepare.sh --modules=cargo-nextest) + # are not broken by the npm commands below. + local any_requested=false + for tool in "${npm_tools[@]}"; do + if contains_module "$tool"; then + any_requested=true + break + fi + done + if [[ "$any_requested" == "false" ]]; then + return 0 + fi - if ! contains_module "$tool"; then + local npm_tools_dir="${SCRIPT_DIR}/npm-tools" + local npm_bin_dir + npm_bin_dir="$(npm config get prefix -g)/bin" + local need_install=false + + for tool in "${npm_tools[@]}"; do + if contains_module "$tool"; then + local expected="${npm_tools_dir}/node_modules/.bin/${tool}" + if [[ "$(readlink "${npm_bin_dir}/${tool}" 2>/dev/null)" != "$expected" ]] || [[ ! -x "$expected" ]]; then + need_install=true + break + fi + fi + done + + if [[ "$need_install" == "false" ]]; then return 0 fi - if [[ "$("$tool" "$version_cmd" 2>/dev/null)" != "$version_pattern" ]]; then - sudo npm install -g "${package}@${version}" + npm ci --prefix "${npm_tools_dir}" + + # Use sudo only when the target directory is not writable (e.g. /usr/local/bin + # on Linux CI runners is root-owned, but Homebrew dirs on macOS are user-owned). + local ln_cmd=(ln -sf) + if [[ ! -w "${npm_bin_dir}" ]]; then + ln_cmd=(sudo ln -sf) fi + for tool in "${npm_tools[@]}"; do + "${ln_cmd[@]}" "${npm_tools_dir}/node_modules/.bin/${tool}" "${npm_bin_dir}/${tool}" + done } # Always ensure git safe.directory is set @@ -221,6 +256,4 @@ maybe_install_cargo_tool dd-rust-license-tool "${DD_RUST_LICENSE_TOOL_VERSION}" maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" -maybe_install_npm_package markdownlint-cli2 markdownlint-cli2 "${MARKDOWNLINT_CLI2_VERSION}" -maybe_install_npm_package prettier prettier "${PRETTIER_VERSION}" -maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" +maybe_install_npm_tools From 04f3b782e95f044fc7307fe4f6230dc48793800b Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Fri, 3 Apr 2026 10:23:45 -0600 Subject: [PATCH 048/364] fix(buffers): Reverse performance regression in buffer metrics (#24995) * fix(buffers): Reverse performance regression in buffer metrics PR #23561 reworked the gauge metric handling for buffer metric reporting in order to get rid of a global shared hashmap. However, in doing so, the current occupancy calculation was moved into the hot path. This change moves that calculation out of the hot path and into the reporting loop, which executes on a fixed interval instead of per event array. * Address PR feedback * Refactor BufferUsage::install and add tests * chore: remove changelog fragment from PR 24995 --------- Co-authored-by: Pavlos Rontidis --- lib/vector-buffers/src/buffer_usage_data.rs | 329 +++++++++++++------- 1 file changed, 215 insertions(+), 114 deletions(-) diff --git a/lib/vector-buffers/src/buffer_usage_data.rs b/lib/vector-buffers/src/buffer_usage_data.rs index ce7a67648db3d..b418023657bb5 100644 --- a/lib/vector-buffers/src/buffer_usage_data.rs +++ b/lib/vector-buffers/src/buffer_usage_data.rs @@ -20,6 +20,7 @@ use crate::{ const ORDERING: Ordering = Ordering::Relaxed; /// Snapshot of category metrics. +#[derive(Clone, Copy, Debug, Default)] struct CategorySnapshot { event_count: u64, event_byte_size: u64, @@ -86,37 +87,52 @@ impl CategoryMetrics { } } -/// `CurrentMetrics` is a wrapper around a pair of `CategoryMetrics` that are used to track a -/// "current" value that may increment or decrement. The challenge this solves is that the -/// increments and decrements may race and result in underflows that are hard to handle using only -/// efficient atomic operations. By tracking the increments and decrements separately, we can ensure -/// that the current value is always accurate even if the increments and decrements race. -#[derive(Debug, Default)] -struct CurrentMetrics { - increments: CategoryMetrics, - decrements: CategoryMetrics, +/// Running totals of events that have entered and left a buffer stage. +/// +/// Each reporting tick consumes the latest deltas from the atomic counters and +/// folds them into these cumulative totals. The difference +/// (`total_entered - total_left`) gives the approximate current buffer +/// occupancy without requiring a separate "current size" counter that would +/// itself be subject to cross-thread races. +#[derive(Clone, Copy, Debug, Default)] +struct ReporterCurrentMetrics { + total_entered: CategorySnapshot, + total_left: CategorySnapshot, } -impl CurrentMetrics { - fn increment(&self, event_count: u64, event_byte_size: u64) { - self.increments.increment(event_count, event_byte_size); +impl ReporterCurrentMetrics { + fn add_received(&mut self, snapshot: CategorySnapshot) { + self.total_entered.event_count = self + .total_entered + .event_count + .saturating_add(snapshot.event_count); + self.total_entered.event_byte_size = self + .total_entered + .event_byte_size + .saturating_add(snapshot.event_byte_size); } - fn decrement(&self, event_count: u64, event_byte_size: u64) { - self.decrements.increment(event_count, event_byte_size); + fn add_left(&mut self, snapshot: CategorySnapshot) { + self.total_left.event_count = self + .total_left + .event_count + .saturating_add(snapshot.event_count); + self.total_left.event_byte_size = self + .total_left + .event_byte_size + .saturating_add(snapshot.event_byte_size); } - fn get(&self) -> CategorySnapshot { - let entered_total = self.increments.get(); - let left_total = self.decrements.get(); - + fn current(&self) -> CategorySnapshot { CategorySnapshot { - event_count: entered_total + event_count: self + .total_entered .event_count - .saturating_sub(left_total.event_count), - event_byte_size: entered_total + .saturating_sub(self.total_left.event_count), + event_byte_size: self + .total_entered .event_byte_size - .saturating_sub(left_total.event_byte_size), + .saturating_sub(self.total_left.event_byte_size), } } } @@ -161,7 +177,6 @@ impl BufferUsageHandle { pub fn increment_received_event_count_and_byte_size(&self, count: u64, byte_size: u64) { if count > 0 || byte_size > 0 { self.state.received.increment(count, byte_size); - self.state.current.increment(count, byte_size); } } @@ -171,7 +186,6 @@ impl BufferUsageHandle { pub fn increment_sent_event_count_and_byte_size(&self, count: u64, byte_size: u64) { if count > 0 || byte_size > 0 { self.state.sent.increment(count, byte_size); - self.state.current.decrement(count, byte_size); } } @@ -188,7 +202,6 @@ impl BufferUsageHandle { } else { self.state.dropped.increment(count, byte_size); } - self.state.current.decrement(count, byte_size); } } } @@ -201,7 +214,6 @@ struct BufferUsageData { dropped: CategoryMetrics, dropped_intentional: CategoryMetrics, max_size: CategoryMetrics, - current: CurrentMetrics, } impl BufferUsageData { @@ -235,6 +247,85 @@ impl BufferUsageData { .expect("should never be bigger than `usize`"), } } + + fn report(&self, current_metrics: &mut ReporterCurrentMetrics, buffer_id: &str) { + let max_size = self.max_size.get(); + emit(BufferCreated { + buffer_id: buffer_id.to_string(), + idx: self.idx, + max_size_bytes: max_size.event_byte_size, + max_size_events: max_size + .event_count + .try_into() + .expect("should never be bigger than `usize`"), + }); + + // Consume received before sent/dropped so that, in the presence of + // a race between producers and consumers, the computed current usage + // will err on the side of overcounting, which is more likely to be an + // accurate representation of the current usage than undercounting. + let received = self.received.consume(); + current_metrics.add_received(received); + + let sent = self.sent.consume(); + current_metrics.add_left(sent); + + let dropped = self.dropped.consume(); + current_metrics.add_left(dropped); + + let dropped_intentional = self.dropped_intentional.consume(); + current_metrics.add_left(dropped_intentional); + + let current = current_metrics.current(); + + if received.has_updates() { + emit(BufferEventsReceived { + buffer_id: buffer_id.to_string(), + idx: self.idx, + count: received.event_count, + byte_size: received.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if sent.has_updates() { + emit(BufferEventsSent { + buffer_id: buffer_id.to_string(), + idx: self.idx, + count: sent.event_count, + byte_size: sent.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if dropped.has_updates() { + emit(BufferEventsDropped { + buffer_id: buffer_id.to_string(), + idx: self.idx, + intentional: false, + reason: "corrupted_events", + count: dropped.event_count, + byte_size: dropped.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if dropped_intentional.has_updates() { + emit(BufferEventsDropped { + buffer_id: buffer_id.to_string(), + idx: self.idx, + intentional: true, + reason: "drop_newest", + count: dropped_intentional.event_count, + byte_size: dropped_intentional.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + } } /// Snapshot of buffer usage metrics. @@ -296,83 +387,29 @@ impl BufferUsage { pub fn install(self, buffer_id: &str) { let buffer_id = buffer_id.to_string(); let span = self.span; - let stages = self.stages; + let stages: Vec<_> = self + .stages + .into_iter() + .map(|stage| (stage, ReporterCurrentMetrics::default())) + .collect(); let task_name = format!("buffer usage reporter ({buffer_id})"); - let task = async move { - let mut interval = interval(Duration::from_secs(2)); - loop { - interval.tick().await; - - for stage in &stages { - let max_size = stage.max_size.get(); - emit(BufferCreated { - buffer_id: buffer_id.clone(), - idx: stage.idx, - max_size_bytes: max_size.event_byte_size, - max_size_events: max_size - .event_count - .try_into() - .expect("should never be bigger than `usize`"), - }); - - let current = stage.current.get(); - let received = stage.received.consume(); - if received.has_updates() { - emit(BufferEventsReceived { - buffer_id: buffer_id.clone(), - idx: stage.idx, - count: received.event_count, - byte_size: received.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let sent = stage.sent.consume(); - if sent.has_updates() { - emit(BufferEventsSent { - buffer_id: buffer_id.clone(), - idx: stage.idx, - count: sent.event_count, - byte_size: sent.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let dropped = stage.dropped.consume(); - if dropped.has_updates() { - emit(BufferEventsDropped { - buffer_id: buffer_id.clone(), - idx: stage.idx, - intentional: false, - reason: "corrupted_events", - count: dropped.event_count, - byte_size: dropped.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let dropped_intentional = stage.dropped_intentional.consume(); - if dropped_intentional.has_updates() { - emit(BufferEventsDropped { - buffer_id: buffer_id.clone(), - idx: stage.idx, - intentional: true, - reason: "drop_newest", - count: dropped_intentional.event_count, - byte_size: dropped_intentional.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - } - } - }; + let task = Self::report_buffer_usage(stages, buffer_id).instrument(span.or_current()); + spawn_named(task, task_name.as_str()); + } - spawn_named(task.instrument(span.or_current()), task_name.as_str()); + async fn report_buffer_usage( + mut stages: Vec<(Arc, ReporterCurrentMetrics)>, + buffer_id: String, + ) { + let mut interval = interval(Duration::from_secs(2)); + loop { + interval.tick().await; + + for (stage, current_metrics) in &mut stages { + stage.report(current_metrics, &buffer_id); + } + } } } @@ -381,27 +418,91 @@ mod tests { use super::*; #[test] - fn current_usage_is_derived_from_entered_and_left_totals() { - let handle = BufferUsageHandle { - state: Arc::new(BufferUsageData::new(0)), - }; - - handle.increment_received_event_count_and_byte_size(10, 1000); - handle.increment_sent_event_count_and_byte_size(3, 300); - handle.increment_dropped_event_count_and_byte_size(2, 200, false); + fn reporter_current_usage_is_derived_from_entered_and_left_totals() { + let mut current = ReporterCurrentMetrics::default(); + current.add_received(CategorySnapshot { + event_count: 10, + event_byte_size: 1000, + }); + current.add_left(CategorySnapshot { + event_count: 3, + event_byte_size: 300, + }); + current.add_left(CategorySnapshot { + event_count: 2, + event_byte_size: 200, + }); + + let current = current.current(); + assert_eq!(current.event_count, 5); + assert_eq!(current.event_byte_size, 500); + } - let current = handle.state.current.get(); + #[test] + fn reporter_current_usage_preserves_underflow_debt() { + let mut current = ReporterCurrentMetrics::default(); + current.add_left(CategorySnapshot { + event_count: 10, + event_byte_size: 1000, + }); + current.add_received(CategorySnapshot { + event_count: 15, + event_byte_size: 1500, + }); + + let current = current.current(); assert_eq!(current.event_count, 5); assert_eq!(current.event_byte_size, 500); } #[test] - fn current_usage_saturates_at_zero() { + fn consume_resets_deltas_between_ticks() { let data = BufferUsageData::new(0); - data.current.decrement(10, 1000); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(10, 1000); + data.sent.increment(3, 300); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + + // Second tick with no new activity should report the same totals. + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + } + + #[test] + fn accumulates_across_multiple_ticks() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(10, 1000); + data.report(&mut metrics, "test"); + + data.received.increment(5, 500); + data.sent.increment(8, 800); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + } + + #[test] + fn drops_count_as_leaving_the_buffer() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(20, 2000); + data.sent.increment(5, 500); + data.dropped.increment(3, 300); + data.dropped_intentional.increment(2, 200); - let current = data.current.get(); - assert_eq!(current.event_count, 0); - assert_eq!(current.event_byte_size, 0); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 10); + assert_eq!(current.event_byte_size, 1000); } } From 51030d2aade15deae23297a1ed1994cb36c530e9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 3 Apr 2026 12:36:36 -0400 Subject: [PATCH 049/364] fix(dev): make file source tests not flaky (#24957) Make file source tests not flaky --- src/sources/file.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/sources/file.rs b/src/sources/file.rs index 62a95db362f10..df51a5a73788a 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -2462,12 +2462,23 @@ mod tests { inner: impl Future, ) -> Vec { assert_source_compliance(&FILE_SOURCE_TAGS, async move { - let (tx, rx) = if acking_mode == Acks { - let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); - (tx, rx.boxed()) - } else { - let (tx, rx) = SourceSender::new_test(); - (tx, rx.boxed()) + let (tx, rx) = match acking_mode { + Acks => { + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); + (tx, rx.boxed()) + } + Unfinalized => { + // Use Rejected so that events are finalized but checkpoints + // are NOT updated (only Delivered triggers checkpoint updates). + // This avoids a race where the default Delivered status on drop + // could leak checkpoint writes into the next run. + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Rejected); + (tx, rx.boxed()) + } + NoAcks => { + let (tx, rx) = SourceSender::new_test(); + (tx, rx.boxed()) + } }; let (trigger_shutdown, shutdown, shutdown_done) = ShutdownSignal::new_wired(); From 2fe515ccd2dfc23d98b84300e4fd87bee27b35fa Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 14:13:12 -0400 Subject: [PATCH 050/364] fix(ci): use helm-charts develop branch for K8s E2E tests (#25118) Instead of pulling the published chart from helm.vector.dev, clone the helm-charts repo (develop branch) and use the local chart path. This breaks the circular dependency between Vector and helm-charts releases and catches chart incompatibilities before release. - Add `helm_chart_repo()` helper with HELM_CHART_REPO env var override - Update deploy-chart-test.sh to support local chart paths - Clone helm-charts in CI and set HELM_CHART_REPO Fixes: #25111 Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/k8s_e2e.yml | 8 ++++++ lib/k8s-e2e-tests/src/lib.rs | 6 ++++ lib/k8s-e2e-tests/tests/vector-agent.rs | 28 +++++++++---------- lib/k8s-e2e-tests/tests/vector-aggregator.rs | 4 +-- .../tests/vector-dd-agent-aggregator.rs | 2 +- lib/k8s-e2e-tests/tests/vector.rs | 8 +++--- scripts/deploy-chart-test.sh | 12 ++++++-- 7 files changed, 44 insertions(+), 24 deletions(-) diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index afdcc01252e20..471783ed1488c 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -197,6 +197,13 @@ jobs: MINIKUBE_VERSION: ${{ matrix.minikube_version }} CONTAINER_RUNTIME: ${{ matrix.container_runtime }} + - name: Checkout helm-charts + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: vectordotdev/helm-charts + ref: develop + path: helm-charts + # TODO: This job has been quite flakey. Need to investigate further and then remove the retries. - name: Run tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 @@ -204,6 +211,7 @@ jobs: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" CARGO_INCREMENTAL: 0 + HELM_CHART_REPO: ${{ github.workspace }}/helm-charts/charts/vector with: timeout_minutes: 45 max_attempts: 3 diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index 4d6d10f4a1875..8986b59ee03a1 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -20,6 +20,12 @@ pub mod metrics; pub const BUSYBOX_IMAGE: &str = "busybox:1.28"; +/// Returns the Helm chart repo to use for E2E tests. +/// Set `HELM_CHART_REPO` to override the default (e.g., a local chart path). +pub fn helm_chart_repo() -> String { + env::var("HELM_CHART_REPO").unwrap_or_else(|_| "https://helm.vector.dev".to_string()) +} + pub fn init() { _ = env_logger::builder().is_test(true).try_init(); } diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 2bf56f32336c6..1a1176b8ce4a1 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -74,7 +74,7 @@ async fn default_agent() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -199,7 +199,7 @@ async fn partial_merge() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -351,7 +351,7 @@ async fn preexisting() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -453,7 +453,7 @@ async fn multiple_lines() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -580,7 +580,7 @@ async fn metadata_annotation() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -769,7 +769,7 @@ async fn pod_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1012,7 +1012,7 @@ async fn custom_selectors() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() @@ -1207,7 +1207,7 @@ async fn container_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1416,7 +1416,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() @@ -1576,7 +1576,7 @@ async fn multiple_ns() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1738,7 +1738,7 @@ async fn existing_config_file() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1839,7 +1839,7 @@ async fn metrics_pipeline() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1994,7 +1994,7 @@ async fn host_metrics() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -2060,7 +2060,7 @@ async fn simple_checkpoint() -> Result<(), Box> { "test-vector", "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![HELM_VALUES_AGENT], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-aggregator.rs index 07cb49ff4bf9d..d945fc5175051 100644 --- a/lib/k8s-e2e-tests/tests/vector-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-aggregator.rs @@ -19,7 +19,7 @@ async fn dummy_topology() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() @@ -54,7 +54,7 @@ async fn metrics_pipeline() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs index 25c7dc4ce6e0a..dde302827cd82 100644 --- a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs @@ -58,7 +58,7 @@ async fn datadog_to_vector() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector.rs b/lib/k8s-e2e-tests/tests/vector.rs index 7d74442fb1146..e38f494236786 100644 --- a/lib/k8s-e2e-tests/tests/vector.rs +++ b/lib/k8s-e2e-tests/tests/vector.rs @@ -77,7 +77,7 @@ async fn logs() -> Result<(), Box> { &namespace, "vector", "aggregator", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { ..Default::default() }, @@ -97,7 +97,7 @@ async fn logs() -> Result<(), Box> { &namespace, "vector", "agent", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&helm_values_stdout_sink(&agent_override_name)], ..Default::default() @@ -198,7 +198,7 @@ async fn haproxy() -> Result<(), Box> { &namespace, "vector", "aggregator", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![CONFIG], ..Default::default() @@ -227,7 +227,7 @@ async fn haproxy() -> Result<(), Box> { &namespace, "vector", "agent", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&helm_values_haproxy(&agent_override_name)], ..Default::default() diff --git a/scripts/deploy-chart-test.sh b/scripts/deploy-chart-test.sh index 343b8030fc202..817161ce41c7b 100755 --- a/scripts/deploy-chart-test.sh +++ b/scripts/deploy-chart-test.sh @@ -61,8 +61,14 @@ up() { # A Vector container image to use. CONTAINER_IMAGE="${CONTAINER_IMAGE:?"You must assign CONTAINER_IMAGE variable with the Vector container image name"}" - $VECTOR_TEST_HELM repo add "$CUSTOM_HELM_REPO_LOCAL_NAME" "$CHART_REPO" --force-update || true - $VECTOR_TEST_HELM repo update + # Support both remote repos (https://...) and local chart paths + if [[ "$CHART_REPO" == http* ]]; then + $VECTOR_TEST_HELM repo add "$CUSTOM_HELM_REPO_LOCAL_NAME" "$CHART_REPO" --force-update || true + $VECTOR_TEST_HELM repo update + HELM_INSTALL_TARGET="$CUSTOM_HELM_REPO_LOCAL_NAME/$HELM_CHART" + else + HELM_INSTALL_TARGET="$CHART_REPO" + fi $VECTOR_TEST_KUBECTL create namespace "$NAMESPACE" --dry-run -o yaml | $VECTOR_TEST_KUBECTL apply -f - @@ -95,7 +101,7 @@ up() { --namespace "$NAMESPACE" \ "${HELM_VALUES[@]}" \ "$RELEASE_NAME" \ - "$CUSTOM_HELM_REPO_LOCAL_NAME/$HELM_CHART" + "$HELM_INSTALL_TARGET" { set +x; } &>/dev/null } From b428aba5dd4f26285ce907212acae648bdb6d8df Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 15:39:14 -0400 Subject: [PATCH 051/364] fix(dev): resolve build.rs HEAD path in worktrees (#25120) * fix(dev): resolve build.rs HEAD path in worktrees * fix(dev): preserve whitespace in git HEAD path --- build.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/build.rs b/build.rs index 5d6d533e86423..01cd4bea5e8ef 100644 --- a/build.rs +++ b/build.rs @@ -104,13 +104,30 @@ fn git_short_hash() -> std::io::Result { }) } +#[cfg(not(feature = "nightly"))] +fn git_path(path: &str) -> std::io::Result { + let output_result = Command::new("git") + .args(["rev-parse", "--git-path", path]) + .output(); + + output_result.map(|output| { + String::from_utf8(output.stdout) + .expect("valid UTF-8") + .trim_end_matches(['\r', '\n']) + .to_owned() + }) +} + fn main() { // Always rerun if the build script itself changes. println!("cargo:rerun-if-changed=build.rs"); // re-run if the HEAD has changed. This is only necessary for non-release and nightly builds. #[cfg(not(feature = "nightly"))] - println!("cargo:rerun-if-changed=.git/HEAD"); + println!( + "cargo:rerun-if-changed={}", + git_path("HEAD").expect("git HEAD path detection failed") + ); #[cfg(feature = "protobuf-build")] { From 738c1d7758024a074f4fbe9ba9fa6d6110b316d7 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 21:14:48 -0400 Subject: [PATCH 052/364] chore(ci): run K8s E2E suite on a weekly schedule (#25119) chore(ci): run K8s E2E suite weekly instead of nightly Reduce the schedule from Tue-Sat to once a week (Monday 01:00 UTC). The suite also runs on PRs that touch K8s-related files and in the merge queue, so weekly is sufficient for catching regressions. Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/k8s_e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 471783ed1488c..5f9ba890ab88f 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -4,7 +4,7 @@ # - manual dispatch in GH UI # - on a PR commit if the kubernetes_logs source was changed # - in the merge queue -# - on a schedule at midnight UTC Tue-Sat +# - on a weekly schedule (Monday 01:00 UTC) # - on demand by either of the following comments in a PR: # - '/ci-run-k8s' # - '/ci-run-all' @@ -34,7 +34,7 @@ on: merge_group: types: [checks_requested] schedule: - - cron: "0 1 * * 2-6" # 01:00 UTC Tue-Sat + - cron: "0 1 * * 1" # 01:00 UTC every Monday concurrency: # In flight runs will be canceled through re-trigger in the merge queue, scheduled run, or if From 703590bff3c2e49aa722c493329c753b1943c4ff Mon Sep 17 00:00:00 2001 From: Caleb Metz <135133572+cmetz100@users.noreply.github.com> Date: Mon, 6 Apr 2026 09:25:55 -0400 Subject: [PATCH 053/364] chore(ci): Migrate smp regression workflow auth from static secrets to oidc (#25112) * migrate smp regression workflow auth to oidc Signed-off-by: Caleb Metz * increase token duration Signed-off-by: Caleb Metz * fmt Signed-off-by: Caleb Metz --------- Signed-off-by: Caleb Metz --- .github/workflows/regression.yml | 39 ++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 167dc0186bada..5a3e1d252b72a 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -278,13 +278,16 @@ jobs: needs: - should-run-gate - resolve-inputs + # Job level permissions for downloading SMP binary + permissions: + id-token: write # Required for GitHub OIDC token exchange steps: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Download SMP binary run: | @@ -303,6 +306,9 @@ jobs: - resolve-inputs - confirm-valid-credentials - build-baseline + # Job level permissions for uploading baseline image to SMP ECR + permissions: + id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download baseline image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 @@ -316,9 +322,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Login to Amazon ECR id: login-ecr @@ -343,6 +349,9 @@ jobs: - resolve-inputs - confirm-valid-credentials - build-comparison + # Job level permissions for uploading comparison image to SMP ECR + permissions: + id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download comparison image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 @@ -356,9 +365,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Login to Amazon ECR id: login-ecr @@ -387,6 +396,7 @@ jobs: permissions: contents: read # Required to checkout code actions: write # Required to upload artifacts + id-token: write # Required for GitHub OIDC token exchange steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -395,9 +405,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Login to Amazon ECR id: login-ecr @@ -462,15 +472,19 @@ jobs: - submit-job - should-run-gate - resolve-inputs + # Job level permissions for downloading SMP results + permissions: + contents: read # Required to checkout code + id-token: write # Required for GitHub OIDC token exchange steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Download SMP binary run: | @@ -503,6 +517,7 @@ jobs: permissions: contents: read # Required to checkout code actions: write # Required to upload artifacts + id-token: write # Required for GitHub OIDC token exchange steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -511,9 +526,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Download SMP binary run: | From 2d6fea2dfe7536202c83305cff5021bfadc90442 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 7 Apr 2026 01:29:11 +1200 Subject: [PATCH 054/364] chore(ci): update check-spelling (#25124) update check-spelling --- .github/workflows/spelling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index e95e1183fc593..3b2c1f57f2d12 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -90,7 +90,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 + uses: check-spelling/check-spelling@a35147f799f30f8739c33f92222c847214e82e67 # v0.0.26 (see check-spelling/check-spelling/issues/103) with: suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} checkout: true From 1152614bc73e74df27986d1bff7864a376025544 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 6 Apr 2026 09:32:37 -0400 Subject: [PATCH 055/364] chore(ci): collect K8s diagnostics on E2E test failure (#25114) Add a diagnostic step to the K8s E2E workflow that runs on failure. Captures pod logs, events, configs, and node resource usage to avoid deep manual investigation when tests fail. Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/k8s_e2e.yml | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 5f9ba890ab88f..1df064c5eebc9 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -164,7 +164,7 @@ jobs: test-e2e-kubernetes: name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }}) runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 60 needs: - build-x86_64-unknown-linux-gnu - compute-k8s-test-plan @@ -217,6 +217,37 @@ jobs: max_attempts: 3 command: make test-e2e-kubernetes + - name: Collect K8s diagnostics on failure + if: ${{ !success() }} + run: | + set +e +o pipefail + # Best-effort diagnostics -- never fail the job + run_diag() { local label="$1"; shift; echo "--- $label ---"; "$@" 2>&1 || true; echo; } + # For commands with pipes that can't be passed as args + run_diag_sh() { echo "--- $1 ---"; bash -c "$2" 2>&1 || true; echo; } + + run_diag "Cluster-wide pods" kubectl get pods -A -o wide + run_diag "Cluster-wide events" kubectl get events -A --sort-by=.metadata.creationTimestamp + run_diag "Nodes" kubectl get nodes -o wide + + for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep -E '^vector-' || true); do + echo "==========================================" + echo "=== Namespace: $ns ===" + echo "==========================================" + run_diag "Pods" kubectl get pods -n "$ns" -o wide + run_diag "Pod descriptions" kubectl describe pods -n "$ns" + run_diag "Events" kubectl get events -n "$ns" --sort-by=.metadata.creationTimestamp + run_diag "ConfigMaps" kubectl get configmaps -n "$ns" -o yaml + + for pod in $(kubectl get pods -n "$ns" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true); do + run_diag "Logs: $pod" kubectl logs -n "$ns" "$pod" --all-containers=true --tail=100 + run_diag "Previous logs: $pod" kubectl logs -n "$ns" "$pod" --all-containers=true --previous --tail=50 + done + done + + run_diag_sh "Node resources" "kubectl describe nodes | grep -A20 'Allocated resources'" + run_diag "Minikube logs" minikube logs --length=100 + final-result: name: K8s E2E Suite runs-on: ubuntu-24.04 From cab8fab16b2da5478d495cd37091b525f527d8ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:37:04 -0400 Subject: [PATCH 056/364] chore(ci): bump the artifact group across 1 directory with 2 updates (#24999) chore(ci): bump the artifact group with 2 updates Bumps the artifact group with 2 updates: [actions/upload-artifact](https://github.com/actions/upload-artifact) and [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/upload-artifact` from 6.0.0 to 7.0.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v6...bbbca2ddaa5d8feaa63e36b76fdaad77386f024f) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: artifact - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: artifact ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/check_generated_vrl_docs.yml | 2 +- .../workflows/check_generated_vrl_docs_comment.yml | 2 +- .github/workflows/integration.yml | 4 ++-- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/publish.yml | 14 +++++++------- .github/workflows/regression.yml | 10 +++++----- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index 7407b28260685..e183e12578f6e 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -112,7 +112,7 @@ jobs: && steps.last-commit.outputs.is-auto != 'true' && github.event_name == 'pull_request' && steps.push.outcome != 'success' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: vrl-docs-check-pr path: /tmp/docs-check/pr-number diff --git a/.github/workflows/check_generated_vrl_docs_comment.yml b/.github/workflows/check_generated_vrl_docs_comment.yml index 25eed42cfeb28..e455203f75a6b 100644 --- a/.github/workflows/check_generated_vrl_docs_comment.yml +++ b/.github/workflows/check_generated_vrl_docs_comment.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write steps: - name: Download PR metadata - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vrl-docs-check-pr run-id: ${{ github.event.workflow_run.id }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 64cfe8c6da004..cdbba4f19187c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -115,7 +115,7 @@ jobs: timeout-minutes: 90 steps: - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 if: github.event_name == 'merge_group' with: name: int_tests_changes @@ -179,7 +179,7 @@ jobs: steps: - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 if: github.event_name == 'merge_group' with: name: e2e_tests_changes diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 1df064c5eebc9..6432b3a8c810e 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -185,7 +185,7 @@ jobs: mold: false cargo-cache: false - - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-test-deb-package path: target/artifacts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4c66ff585e173..239b7f95388f6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -221,7 +221,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -272,7 +272,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -301,7 +301,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (${{ matrix.target }}) - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts @@ -351,7 +351,7 @@ jobs: with: version: latest - name: Download all Linux package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-*-linux-* path: target/artifacts @@ -394,7 +394,7 @@ jobs: with: vdev: true - name: Download all package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-* path: target/artifacts @@ -434,7 +434,7 @@ jobs: with: vdev: true - name: Download all package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-* path: target/artifacts @@ -461,7 +461,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download all package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-* path: target/artifacts diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5a3e1d252b72a..cc1170a8aaa21 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -311,7 +311,7 @@ jobs: id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download baseline image" - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: baseline-image @@ -354,7 +354,7 @@ jobs: id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download comparison image" - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: comparison-image @@ -491,7 +491,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -535,7 +535,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -581,7 +581,7 @@ jobs: steps: - name: Download capture-artifacts continue-on-error: true - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: capture-artifacts From 4b2353a35dd54b888cdee7d1ca70c8723e12873c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:46:52 -0400 Subject: [PATCH 057/364] chore(ci): bump nick-fields/retry from 3.0.2 to 4.0.0 (#25102) Bumps [nick-fields/retry](https://github.com/nick-fields/retry) from 3.0.2 to 4.0.0. - [Release notes](https://github.com/nick-fields/retry/releases) - [Commits](https://github.com/nick-fields/retry/compare/ce71cc2ab81d554ebbe88c79ab5975992d79ba08...ad984534de44a9489a53aefd81eb77f87c70dc60) --- updated-dependencies: - dependency-name: nick-fields/retry dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-integration-review.yml | 4 ++-- .github/workflows/integration.yml | 4 ++-- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/unit_mac.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 67fae9ea864d7..77b20b347e318 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -174,7 +174,7 @@ jobs: - name: Integration Tests - ${{ matrix.service }} if: steps.run_condition.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 30 max_attempts: 3 @@ -228,7 +228,7 @@ jobs: - name: E2E Tests - ${{ matrix.service }} if: steps.run_condition.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 35 max_attempts: 3 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cdbba4f19187c..2c1e91c57a8e7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -158,7 +158,7 @@ jobs: - name: Run Integration Tests for ${{ matrix.service }} if: steps.check.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 30 max_attempts: 3 @@ -222,7 +222,7 @@ jobs: - name: Run E2E Tests for ${{ matrix.service }} if: steps.check.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 30 max_attempts: 3 diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 6432b3a8c810e..3b35ec0b7b9dc 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -206,7 +206,7 @@ jobs: # TODO: This job has been quite flakey. Need to investigate further and then remove the retries. - name: Run tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 env: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 239b7f95388f6..d5093f229017c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -363,7 +363,7 @@ jobs: env: PLATFORM: "linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6" REPOS: "timberio/vector,ghcr.io/vectordotdev/vector" - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 15 max_attempts: 3 diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index e0d367b530959..a44a84d368953 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -31,7 +31,7 @@ jobs: # Some tests e.g. `reader_exits_cleanly_when_writer_done_and_in_flight_acks` are flaky. - name: Run tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 45 max_attempts: 3 From 9e897388fbf772a23c48df93de923a05b2d3ecbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 20:05:11 +0000 Subject: [PATCH 058/364] chore(deps): bump fast-xml-parser from 5.3.7 to 5.5.9 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory (#25127) chore(deps): bump fast-xml-parser Bumps the npm_and_yarn group with 1 update in the /scripts/environment/npm-tools directory: [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser). Updates `fast-xml-parser` from 5.3.7 to 5.5.9 - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.3.7...v5.5.9) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.5.9 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- .../environment/npm-tools/package-lock.json | 192 ++++++++++-------- scripts/environment/npm-tools/package.json | 2 +- 2 files changed, 113 insertions(+), 81 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index e3d4efa122515..1047d5444f89a 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -5,7 +5,7 @@ "packages": { "": { "dependencies": { - "@datadog/datadog-ci": "5.11.0", + "@datadog/datadog-ci": "5.12.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } @@ -24,19 +24,19 @@ } }, "node_modules/@datadog/datadog-ci": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.11.0.tgz", - "integrity": "sha512-4N/WsfnQVYJNxGtDIOhg4sUkZalDo8h+toSXR/GAw6XU81e1bOj3GvleL6IHUQ06KyPdurqPZKB4Za1CbCsT7g==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.12.0.tgz", + "integrity": "sha512-8zJOgx0ouVZtspXTGdx327Yt6UiRnaQSlMc7c343a1HaKC9nIlQhnd90G1frFZhopiqA1KD8h9CWBhygxAJW7w==", "license": "Apache-2.0", "dependencies": { - "@datadog/datadog-ci-base": "5.11.0", - "@datadog/datadog-ci-plugin-coverage": "5.11.0", - "@datadog/datadog-ci-plugin-deployment": "5.11.0", - "@datadog/datadog-ci-plugin-dora": "5.11.0", - "@datadog/datadog-ci-plugin-gate": "5.11.0", - "@datadog/datadog-ci-plugin-junit": "5.11.0", - "@datadog/datadog-ci-plugin-sarif": "5.11.0", - "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "@datadog/datadog-ci-base": "5.12.0", + "@datadog/datadog-ci-plugin-coverage": "5.12.0", + "@datadog/datadog-ci-plugin-deployment": "5.12.0", + "@datadog/datadog-ci-plugin-dora": "5.12.0", + "@datadog/datadog-ci-plugin-gate": "5.12.0", + "@datadog/datadog-ci-plugin-junit": "5.12.0", + "@datadog/datadog-ci-plugin-sarif": "5.12.0", + "@datadog/datadog-ci-plugin-sbom": "5.12.0", "clipanion": "3.2.1", "typanion": "3.14.0" }, @@ -48,23 +48,23 @@ } }, "node_modules/@datadog/datadog-ci-base": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.11.0.tgz", - "integrity": "sha512-NACS6zhf2qidgpgrhQH0EJn4x20pwyRAhw75jBRPmjzheaXrpU0438e3kutyNsDGZrw8gHjuRqiccoY3QU7PSQ==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.12.0.tgz", + "integrity": "sha512-ZyobD2nbzwRksWiYvcxl3ZuYE4+AX1is26qjWAs1ufhVyGBCx1cVqiDFEAILeyqkncZYIFGppjCyjUcxev+CBg==", "license": "Apache-2.0", "dependencies": { "@antfu/install-pkg": "1.1.0", "@types/datadog-metrics": "0.6.1", - "async-retry": "1.3.1", + "async-retry": "1.3.3", "axios": "1.13.5", "chalk": "3.0.0", "clipanion": "3.2.1", "datadog-metrics": "0.9.3", "debug": "4.4.1", "deep-extend": "0.6.0", - "fast-xml-parser": "5.3.7", + "fast-xml-parser": "5.5.9", "form-data": "4.0.4", - "glob": "13.0.5", + "glob": "13.0.6", "inquirer": "8.2.7", "is-docker": "4.0.0", "jest-diff": "30.2.0", @@ -79,17 +79,17 @@ "upath": "2.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-plugin-aas": "5.11.0", - "@datadog/datadog-ci-plugin-cloud-run": "5.11.0", - "@datadog/datadog-ci-plugin-container-app": "5.11.0", - "@datadog/datadog-ci-plugin-coverage": "5.11.0", - "@datadog/datadog-ci-plugin-deployment": "5.11.0", - "@datadog/datadog-ci-plugin-dora": "5.11.0", - "@datadog/datadog-ci-plugin-gate": "5.11.0", - "@datadog/datadog-ci-plugin-junit": "5.11.0", - "@datadog/datadog-ci-plugin-lambda": "5.11.0", - "@datadog/datadog-ci-plugin-sarif": "5.11.0", - "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "@datadog/datadog-ci-plugin-aas": "5.12.0", + "@datadog/datadog-ci-plugin-cloud-run": "5.12.0", + "@datadog/datadog-ci-plugin-container-app": "5.12.0", + "@datadog/datadog-ci-plugin-coverage": "5.12.0", + "@datadog/datadog-ci-plugin-deployment": "5.12.0", + "@datadog/datadog-ci-plugin-dora": "5.12.0", + "@datadog/datadog-ci-plugin-gate": "5.12.0", + "@datadog/datadog-ci-plugin-junit": "5.12.0", + "@datadog/datadog-ci-plugin-lambda": "5.12.0", + "@datadog/datadog-ci-plugin-sarif": "5.12.0", + "@datadog/datadog-ci-plugin-sbom": "5.12.0", "@datadog/datadog-ci-plugin-stepfunctions": "5.11.0", "@datadog/datadog-ci-plugin-synthetics": "5.11.0", "@datadog/datadog-ci-plugin-terraform": "5.11.0" @@ -157,24 +157,24 @@ } }, "node_modules/@datadog/datadog-ci-plugin-coverage": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.11.0.tgz", - "integrity": "sha512-aKzkjbd/NVt+7Sd4QL0qVqUrnyMG0UF63lvNxve7suWQDoIWowAgCPa0ta+jpMYUYCGtjt74H3wL+q0W0G+D+Q==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.12.0.tgz", + "integrity": "sha512-ll4aBYF+FMOlOjAnNiPjO8iPmVB/qfBHrUOCk2EMnOaQn6Eve/FajCaSqeV2BoI9+3UXv04P6eH5nxPtpcM/cg==", "license": "Apache-2.0", "dependencies": { "chalk": "3.0.0", - "fast-xml-parser": "5.3.7", + "fast-xml-parser": "5.5.9", "form-data": "4.0.4", "upath": "2.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-deployment": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.11.0.tgz", - "integrity": "sha512-FF9L7alDz1XN5L3nYR7xMeZd99U6kxkT+InSyZY21ZHpcuuEngseW1aMgB0iFn3lxZ+1H4z196uCckViTEkjog==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.12.0.tgz", + "integrity": "sha512-6KDwUmdxwGxYoBEYqAY0uo7ocvtTq3p3cXW4KjpDXwZyr1wWstzXQxTx7qquZzMMNvsD6JCq+HDqwdTgaHSAog==", "license": "Apache-2.0", "dependencies": { "axios": "1.13.5", @@ -182,26 +182,26 @@ "simple-git": "3.33.0" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-dora": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.11.0.tgz", - "integrity": "sha512-DWSyT2tW23XOYCvTEHvD/F9FRIffTN+2wZGYojaD1LM9wJew4xievziBwyD46quCF0OdsbGMLvAyR2hyrrYeZA==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.12.0.tgz", + "integrity": "sha512-HrC6H5lLzRTv3tKiS2f5YMU/WWwgzMF2kHIJU6+pe0y/v1xUYCELV112XYldbg3dkENfpG/AwjUeO6B8rE3s+g==", "license": "Apache-2.0", "dependencies": { "chalk": "3.0.0", "simple-git": "3.33.0" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-gate": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.11.0.tgz", - "integrity": "sha512-WUNr6y292eH+sumAyD5rFhqlU/aJ8vN74h4EJj0SqDpnrDCzMnSDgRyb9GXhv8ZO4GOW6LTEcPmMZhyfYL3b5g==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.12.0.tgz", + "integrity": "sha512-YNZuRigmGDGK2tK6xRsaq0PPx/4CRVZ4U6q7xabIP+jmhoG7mbI1RFUjlXSnGfeU1SXm0bVQw8F69AcZCHtNOA==", "license": "Apache-2.0", "dependencies": { "@types/uuid": "9.0.8", @@ -209,29 +209,29 @@ "uuid": "9.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-junit": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.11.0.tgz", - "integrity": "sha512-I8H09jNR/LDYqiy7TOKv9Gu4+t/o0J6QH6cooKrNboULSDYijVo6w3knANsd2m3Hlcbi1QkTmNpPXIdeGR5Dig==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.12.0.tgz", + "integrity": "sha512-XPVULYU1RUN2kFmSWGfAgU+psGFhPu+5NxnHeAQubs/wYcGwdZUEAD2+tn4F3PxpCkTduEb4FO6mLpYbjBpuqA==", "license": "Apache-2.0", "dependencies": { "chalk": "3.0.0", - "fast-xml-parser": "5.3.7", + "fast-xml-parser": "5.5.9", "form-data": "4.0.4", "upath": "2.0.1", "uuid": "9.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-sarif": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.11.0.tgz", - "integrity": "sha512-JZuqNzFsW9BZeStLM60f5zdEpG67kh7xqgbeJhqIcloEpDJIEmsJHH487PM0stakcqhhIfzUzvEW744LLQg5Ug==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.12.0.tgz", + "integrity": "sha512-tmUEv6sJWsFG9vnriSu9RZSrLvYo0EfvRaHiiO3ufxBC+NvU0HZYhtTqXVT6NnsUSVv/2FBnzl0hCnPDHhyBFg==", "license": "Apache-2.0", "dependencies": { "ajv": "8.18.0", @@ -242,13 +242,13 @@ "uuid": "9.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-sbom": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.11.0.tgz", - "integrity": "sha512-lHU65a1Hru7SiwhZtZ9OTd6vyPcRG2U/61imVpvLvhEMSwY2xSLTtZfxWbMi4VJxq81V2zhdifxP5BMJO1FnHw==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.12.0.tgz", + "integrity": "sha512-F0c0gfX4O+Qm1heLEhNCTl/tUPlVQvyFGYTUqJ0Z2niqT4Y3z7Oi6RSodgUSwGCKl6bwacRjTTdCrzI3V0n3nA==", "license": "Apache-2.0", "dependencies": { "ajv": "8.18.0", @@ -258,7 +258,7 @@ "packageurl-js": "2.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@inquirer/external-editor": { @@ -525,12 +525,12 @@ } }, "node_modules/async-retry": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", - "integrity": "sha512-aiieFW/7h3hY0Bq5d+ktDBejxuwR78vRu9hDUdR8rNhSaQ29VzPL4AoIRG7D/c7tdenwOcKvgPM6tIxB3cB6HA==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", "license": "MIT", "dependencies": { - "retry": "0.12.0" + "retry": "0.13.1" } }, "node_modules/asynckit": { @@ -1178,10 +1178,25 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } + }, "node_modules/fast-xml-parser": { - "version": "5.3.7", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.7.tgz", - "integrity": "sha512-JzVLro9NQv92pOM/jTCR6mHlJh2FGwtomH8ZQjhFj/R29P2Fnj38OgPJVtcvYw6SuKClhgYuwUZf5b3rd8u2mA==", + "version": "5.5.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.9.tgz", + "integrity": "sha512-jldvxr1MC6rtiZKgrFnDSvT8xuH+eJqxqOBThUVjYrxssYTo1avZLGql5l0a0BAERR01CadYzZ83kVEkbyDg+g==", "funding": [ { "type": "github", @@ -1190,7 +1205,9 @@ ], "license": "MIT", "dependencies": { - "strnum": "^2.1.2" + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.2.0", + "strnum": "^2.2.2" }, "bin": { "fxparser": "src/cli/cli.js" @@ -1341,17 +1358,17 @@ } }, "node_modules/glob": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.5.tgz", - "integrity": "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw==", + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", "license": "BlueOak-1.0.0", "dependencies": { - "minimatch": "^10.2.1", - "minipass": "^7.1.2", - "path-scurry": "^2.0.0" + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1884,9 +1901,9 @@ } }, "node_modules/lru-cache": { - "version": "11.2.7", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", - "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.0.tgz", + "integrity": "sha512-sr8xPKE25m6vJVcrdn6NxtC0fVfuPowbscLypegRgOm0yXSqr5JNHCAY3hnusdJ7HRBW04j6Ip4khvHU778DuQ==", "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" @@ -2768,6 +2785,21 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/path-expression-matcher": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.2.1.tgz", + "integrity": "sha512-d7gQQmLvAKXKXE2GeP9apIGbMYKz88zWdsn/BN2HRWVQsDFdUY36WSLTY0Jvd4HWi7Fb30gQ62oAOzdgJA6fZw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-scurry": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", @@ -2965,9 +2997,9 @@ } }, "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", "license": "MIT", "engines": { "node": ">= 4" diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json index 30d63057b7fed..a7ec2311e71d2 100644 --- a/scripts/environment/npm-tools/package.json +++ b/scripts/environment/npm-tools/package.json @@ -2,7 +2,7 @@ "private": true, "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", "dependencies": { - "@datadog/datadog-ci": "5.11.0", + "@datadog/datadog-ci": "5.12.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } From 0c5fd58228bfab0dd9f6e81a1dc651da2ea8e670 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 7 Apr 2026 12:14:58 -0400 Subject: [PATCH 059/364] chore(api top): improve dashboard column layout (#25135) enhancement(vector top): improve dashboard column layout - Move Errors column to always be last (after Memory Used when allocation-tracing is enabled) - Slightly increase Kind column width (+1%) and decrease Errors (-1%) Co-authored-by: Claude Opus 4.6 (1M context) --- lib/vector-top/src/dashboard.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index 1b73d5b89e9c8..8f54c48534f71 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -170,9 +170,9 @@ static HEADER: [&str; NUM_COLUMNS] = [ columns::BYTES_IN, columns::EVENTS_OUT, columns::BYTES_OUT, - columns::ERRORS, #[cfg(feature = "allocation-tracing")] columns::MEMORY_USED, + columns::ERRORS, ]; struct Widgets<'a> { @@ -348,13 +348,13 @@ impl<'a> Widgets<'a> { r.sent_bytes_throughput_sec, self.human_metrics, ), + #[cfg(feature = "allocation-tracing")] + r.allocated_bytes.human_format_bytes(), if self.human_metrics { r.errors.human_format() } else { r.errors.thousands_format() }, - #[cfg(feature = "allocation-tracing")] - r.allocated_bytes.human_format_bytes(), ]; data.extend_from_slice(&formatted_metrics); @@ -383,26 +383,26 @@ impl<'a> Widgets<'a> { &[ Constraint::Percentage(13), // ID Constraint::Percentage(8), // Output - Constraint::Percentage(4), // Kind + Constraint::Percentage(5), // Kind Constraint::Percentage(9), // Type Constraint::Percentage(10), // Events In Constraint::Percentage(12), // Bytes In Constraint::Percentage(10), // Events Out Constraint::Percentage(12), // Bytes Out - Constraint::Percentage(8), // Errors - Constraint::Percentage(14), // Allocated Bytes + Constraint::Percentage(14), // Memory Used + Constraint::Percentage(7), // Errors ] } else { &[ Constraint::Percentage(13), // ID Constraint::Percentage(12), // Output - Constraint::Percentage(9), // Kind + Constraint::Percentage(10), // Kind Constraint::Percentage(6), // Type Constraint::Percentage(12), // Events In Constraint::Percentage(14), // Bytes In Constraint::Percentage(12), // Events Out Constraint::Percentage(14), // Bytes Out - Constraint::Percentage(8), // Errors + Constraint::Percentage(7), // Errors ] }; let w = Table::new(items, widths) From 02f281d6ad2786f8bbfad9ece820b8d9f4925571 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 7 Apr 2026 15:38:29 -0400 Subject: [PATCH 060/364] enhancement(api top): show "disabled" in Memory Used when allocation tracing is off (#25138) * enhancement(vector top): show "disabled" in Memory Used when allocation tracing is off Add GetAllocationTracingStatus gRPC endpoint that reports whether the connected Vector instance has --allocation-tracing active. vector-top queries this on startup and shows "disabled" in the Memory Used column when tracing is off, instead of misleading zeros. Changes: - proto: add GetAllocationTracingStatus RPC and messages - service.rs: implement the endpoint using is_allocation_tracing_enabled() - client.rs: add get_allocation_tracing_status() method - metrics.rs: query the endpoint on startup, store result in State - state.rs: add allocation_tracing_active field to State - dashboard.rs: show "disabled" when allocation_tracing_active is false Co-Authored-By: Claude Opus 4.6 (1M context) * chore: add changelog fragment for Memory Used disabled label Co-Authored-By: Claude Opus 4.6 (1M context) * add authors * fix(api): guard allocation tracing RPC behind cfg feature flag The `get_allocation_tracing_status` RPC handler unconditionally called `is_allocation_tracing_enabled()`, which only exists behind the `allocation-tracing` feature gate. This broke builds on feature sets that lack it (e.g. `default-msvc`). Add cfg guards so the handler returns `false` when the feature is absent. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...tor_top_disabled_memory_used.enhancement.md | 3 +++ lib/vector-api-client/src/client.rs | 12 ++++++++++++ lib/vector-top/src/dashboard.rs | 6 +++++- lib/vector-top/src/metrics.rs | 18 +++++++++++++++++- lib/vector-top/src/state.rs | 6 ++++++ proto/vector/observability.proto | 9 +++++++++ src/api/grpc/service.rs | 13 +++++++++++++ 7 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 changelog.d/vector_top_disabled_memory_used.enhancement.md diff --git a/changelog.d/vector_top_disabled_memory_used.enhancement.md b/changelog.d/vector_top_disabled_memory_used.enhancement.md new file mode 100644 index 0000000000000..e929a065019ef --- /dev/null +++ b/changelog.d/vector_top_disabled_memory_used.enhancement.md @@ -0,0 +1,3 @@ +`vector top` terminal UI now shows `disabled` in the Memory Used column when the connected Vector instance was not started with `--allocation-tracing`, instead of displaying misleading zeros. A new `GetAllocationTracingStatus` gRPC endpoint is queried on connect to determine the status. + +authors: pront diff --git a/lib/vector-api-client/src/client.rs b/lib/vector-api-client/src/client.rs index d4991efef0894..fcba2702a3e2a 100644 --- a/lib/vector-api-client/src/client.rs +++ b/lib/vector-api-client/src/client.rs @@ -5,6 +5,7 @@ use tonic::transport::{Channel, Endpoint}; use crate::{ error::{Error, Result}, proto::{ + GetAllocationTracingStatusRequest, GetAllocationTracingStatusResponse, GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, HealthRequest, HealthResponse, MetricName, StreamComponentAllocatedBytesRequest, StreamComponentAllocatedBytesResponse, StreamComponentMetricsRequest, @@ -77,6 +78,17 @@ impl Client { Ok(response.into_inner()) } + /// Check whether allocation tracing is active on the connected Vector instance + pub async fn get_allocation_tracing_status( + &mut self, + ) -> Result { + let client = self.ensure_connected()?; + let response = client + .get_allocation_tracing_status(GetAllocationTracingStatusRequest {}) + .await?; + Ok(response.into_inner()) + } + // ========== Streaming RPCs ========== /// Stream periodic heartbeat timestamps diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index 8f54c48534f71..9e6a1907dff36 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -349,7 +349,11 @@ impl<'a> Widgets<'a> { self.human_metrics, ), #[cfg(feature = "allocation-tracing")] - r.allocated_bytes.human_format_bytes(), + if state.allocation_tracing_active { + r.allocated_bytes.human_format_bytes() + } else { + "disabled".to_string() + }, if self.human_metrics { r.errors.human_format() } else { diff --git a/lib/vector-top/src/metrics.rs b/lib/vector-top/src/metrics.rs index 52cf46beff244..63328a62825f0 100644 --- a/lib/vector-top/src/metrics.rs +++ b/lib/vector-top/src/metrics.rs @@ -569,5 +569,21 @@ pub async fn init_components( }) .collect::>(); - Ok(state::State::new(rows)) + let mut state = state::State::new(rows); + + #[cfg(feature = "allocation-tracing")] + { + // Allocation tracing is a compile-time + startup-time setting on the + // server, so querying once per connection is sufficient. On error + // (e.g. older server without this RPC) we default to false, matching + // pre-existing behavior. This is re-evaluated on every reconnect via + // the retry loop in `subscription()`. + state.allocation_tracing_active = client + .get_allocation_tracing_status() + .await + .map(|r| r.enabled) + .unwrap_or(false); + } + + Ok(state) } diff --git a/lib/vector-top/src/state.rs b/lib/vector-top/src/state.rs index a954c804ce467..30fc05ef76c13 100644 --- a/lib/vector-top/src/state.rs +++ b/lib/vector-top/src/state.rs @@ -122,6 +122,10 @@ pub struct State { pub sort_state: SortState, pub filter_state: FilterState, pub ui: UiState, + /// Set to `true` once we receive the first `AllocatedBytes` event, + /// indicating the connected Vector instance has allocation tracing active. + #[cfg(feature = "allocation-tracing")] + pub allocation_tracing_active: bool, } #[derive(Debug, Clone, Copy)] @@ -343,6 +347,8 @@ impl State { ui: UiState::default(), sort_state: SortState::default(), filter_state: FilterState::default(), + #[cfg(feature = "allocation-tracing")] + allocation_tracing_active: false, } } diff --git a/proto/vector/observability.proto b/proto/vector/observability.proto index 1c1d508e5d26c..9996a47908949 100644 --- a/proto/vector/observability.proto +++ b/proto/vector/observability.proto @@ -17,6 +17,9 @@ service ObservabilityService { // Get information about configured components (sources, transforms, sinks) rpc GetComponents(GetComponentsRequest) returns (GetComponentsResponse); + // Check whether allocation tracing is active on this instance + rpc GetAllocationTracingStatus(GetAllocationTracingStatusRequest) returns (GetAllocationTracingStatusResponse); + // ========== Real-time Metric Streams ========== // All streaming RPCs send periodic updates at the specified interval @@ -54,6 +57,12 @@ message GetMetaResponse { string hostname = 2; } +message GetAllocationTracingStatusRequest {} + +message GetAllocationTracingStatusResponse { + bool enabled = 1; +} + // ========== Component Messages ========== message GetComponentsRequest { diff --git a/src/api/grpc/service.rs b/src/api/grpc/service.rs index 171f65853cc91..580ebb69b5219 100644 --- a/src/api/grpc/service.rs +++ b/src/api/grpc/service.rs @@ -436,6 +436,19 @@ impl observability::Service for ObservabilityService { Ok(Response::new(GetMetaResponse { version, hostname })) } + async fn get_allocation_tracing_status( + &self, + _request: Request, + ) -> Result, Status> { + #[cfg(feature = "allocation-tracing")] + let enabled = crate::internal_telemetry::allocations::is_allocation_tracing_enabled(); + #[cfg(not(feature = "allocation-tracing"))] + let enabled = false; + Ok(Response::new(GetAllocationTracingStatusResponse { + enabled, + })) + } + async fn get_components( &self, request: Request, From 256db9c12951987ea235a0c5a4a88898d5187642 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 8 Apr 2026 11:32:50 -0400 Subject: [PATCH 061/364] fix(api top): avoid unused-mut warning on Windows builds (#25142) The `allocation-tracing` feature is gated behind `unix`, so on Windows `state` was never mutated, triggering `-D unused-mut`. Split into two cfg branches so each path only declares what it needs. Co-authored-by: Claude Opus 4.6 (1M context) --- lib/vector-top/src/metrics.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/vector-top/src/metrics.rs b/lib/vector-top/src/metrics.rs index 63328a62825f0..75aae4456fce6 100644 --- a/lib/vector-top/src/metrics.rs +++ b/lib/vector-top/src/metrics.rs @@ -569,8 +569,6 @@ pub async fn init_components( }) .collect::>(); - let mut state = state::State::new(rows); - #[cfg(feature = "allocation-tracing")] { // Allocation tracing is a compile-time + startup-time setting on the @@ -578,12 +576,15 @@ pub async fn init_components( // (e.g. older server without this RPC) we default to false, matching // pre-existing behavior. This is re-evaluated on every reconnect via // the retry loop in `subscription()`. + let mut state = state::State::new(rows); state.allocation_tracing_active = client .get_allocation_tracing_status() .await .map(|r| r.enabled) .unwrap_or(false); + Ok(state) } - Ok(state) + #[cfg(not(feature = "allocation-tracing"))] + Ok(state::State::new(rows)) } From 711b039fe7a8ddac2296dfe1e78685bb901795e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 13:30:51 -0400 Subject: [PATCH 062/364] chore(website deps): bump lodash from 4.17.23 to 4.18.1 in /website (#25113) Bumps [lodash](https://github.com/lodash/lodash) from 4.17.23 to 4.18.1. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.23...4.18.1) --- updated-dependencies: - dependency-name: lodash dependency-version: 4.18.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index da88eb1b5980f..055e51daa030a 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2815,9 +2815,9 @@ lodash.uniq@^4.5.0: integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== lodash@^4.17.21: - version "4.17.23" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.23.tgz#f113b0378386103be4f6893388c73d0bde7f2c5a" - integrity sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w== + version "4.18.1" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.18.1.tgz#ff2b66c1f6326d59513de2407bf881439812771c" + integrity sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q== logform@^2.6.0, logform@^2.6.1: version "2.6.1" From dcf98a7ccf54080a74626e9277ac5b2f8340ded8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 8 Apr 2026 13:37:57 -0400 Subject: [PATCH 063/364] chore: fix spell check CI failures (#25144) chore: fix spell check failures for gzip'd and schema'd Replace `gzip'd` with `gzip'ed` (already in the allow list) and `schema'd` with `schema-based` to fix the Check Spelling CI job. Co-authored-by: Claude Opus 4.6 (1M context) --- rfcs/2021-01-08-5843-encoding-decoding-format-vector.md | 2 +- website/content/en/guides/aws/cloudwatch-logs-firehose.md | 2 +- website/cue/reference/releases/0.28.0.cue | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md b/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md index 9687c50e06992..59bfd92566555 100644 --- a/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md +++ b/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md @@ -248,7 +248,7 @@ Per the data format itself - I haven't found any single conglomerate conclusion ## Drawbacks -The only drawback to adopting protobuf as the encoding format is that Protobufs can be slightly slower than other schema'd data formats and we don't shed the protobuf tooling overhead. +The only drawback to adopting protobuf as the encoding format is that Protobufs can be slightly slower than other schema-based data formats and we don't shed the protobuf tooling overhead. Drawbacks to adopting HTTP/2 are that (like literally any other transport we could pick) it alters our kubernetes deployment handling and if we don't decide to continue to maintain other transports it means needing to appropriately handle the deprecation and removal of our TCP and HTTP/1.1 implementations fairly soon afterwards. diff --git a/website/content/en/guides/aws/cloudwatch-logs-firehose.md b/website/content/en/guides/aws/cloudwatch-logs-firehose.md index 8323be561df9c..02c4e48b40146 100644 --- a/website/content/en/guides/aws/cloudwatch-logs-firehose.md +++ b/website/content/en/guides/aws/cloudwatch-logs-firehose.md @@ -457,7 +457,7 @@ The [`aws_kinesis_firehose`][aws_kinesis_firehose] source: access_key = "my secret key" # this will also be set in the Firehose config ``` -will accept this request, decode the record (which is gzip'd and then base64 encoded), to produce an event that looks like: +will accept this request, decode the record (which is gzip'ed and then base64 encoded), to produce an event that looks like: ```json { diff --git a/website/cue/reference/releases/0.28.0.cue b/website/cue/reference/releases/0.28.0.cue index 91032cae68694..7d44ba72e215b 100644 --- a/website/cue/reference/releases/0.28.0.cue +++ b/website/cue/reference/releases/0.28.0.cue @@ -144,7 +144,7 @@ releases: "0.28.0": { type: "feat" scopes: ["vrl: stdlib"] description: """ - `encode_gzip` and `decode_gzip` functions were added to VRL to interact with gzip'd + `encode_gzip` and `decode_gzip` functions were added to VRL to interact with gzip'ed data. """ contributors: ["zamazan4ik"] From 30d9a58669556a9bc014ba0b2ea4f33f8e372d98 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 9 Apr 2026 05:43:50 +1200 Subject: [PATCH 064/364] feat(azure_blob sink): Expand support for Azure authentication types (#24729) * start by moving auth config to azure_common * add auth to azure_blob Signed-off-by: Jed Laundry * initial auth switch Signed-off-by: Jed Laundry * integration test WIP Signed-off-by: Jed Laundry * doc updates Signed-off-by: Jed Laundry * add feature changelog Signed-off-by: Jed Laundry * clippy changes Signed-off-by: Jed Laundry * add account_name and blob_endpoint options Signed-off-by: Jed Laundry * remove block_in_place * add ClientCertificateCredential Signed-off-by: Jed Laundry * remake certificate with required attributes * refactor integration tests to support both connection string and oauth Signed-off-by: Jed Laundry * update doc examples Signed-off-by: Jed Laundry * make clippy happy Signed-off-by: Jed Laundry * fix tests Signed-off-by: Jed Laundry * make "client_secret_credential" explicitly configured Signed-off-by: Jed Laundry * add azure_logs_ingestion to Azure integration tests Signed-off-by: Jed Laundry * Err out the other TlsConfig options Signed-off-by: Jed Laundry * Add warning message to connection_string Signed-off-by: Jed Laundry * Simplify TlsConfig Signed-off-by: Jed Laundry * add workload identity config options * Move warning into docs::warnings * Properly regenerate cue file * Fix spaces * tidy options creation Signed-off-by: Jed Laundry * add UAMI type option Signed-off-by: Jed Laundry * add note about breaking change * propagate credential errors instead of panicking * automatic Cargo lock change Signed-off-by: Jed Laundry * clarify PEM cert file Signed-off-by: Jed Laundry * action test fixes Signed-off-by: Jed Laundry * allow sinks-azure_logs_ingestion to build without sinks-azure_blob Signed-off-by: Jed Laundry * require auth when using account_name or blob_endpoint Signed-off-by: Jed Laundry * add dep:base64 Signed-off-by: Jed Laundry * check-fmt changes * space diff fix --------- Signed-off-by: Jed Laundry Co-authored-by: Thomas Co-authored-by: Pavlos Rontidis --- .github/actions/spelling/allow.txt | 2 + .github/actions/spelling/expect.txt | 1 + Cargo.lock | 1 + Cargo.toml | 10 +- ...24729_azure_blob_authentication.feature.md | 3 + .../24729_azure_logs_ingestion.breaking.md | 3 + src/sinks/azure_blob/config.rs | 88 ++- src/sinks/azure_blob/integration_tests.rs | 174 ++++-- src/sinks/azure_blob/test.rs | 320 +++++++++++ src/sinks/azure_common/config.rs | 534 +++++++++++++++++- src/sinks/azure_logs_ingestion/config.rs | 200 +------ src/sinks/azure_logs_ingestion/tests.rs | 62 +- src/sinks/mod.rs | 2 +- tests/data/ClientCertificateAuth.pfx | Bin 0 -> 2467 bytes tests/data/Makefile | 21 + .../certs/azurite-chain.cert.pem | 98 ++++ .../certs/azurite.cert.pem | 32 ++ .../intermediate_server/csr/azurite.csr.pem | 18 + tests/data/ca/intermediate_server/index.txt | 1 + .../data/ca/intermediate_server/index.txt.old | 1 + .../ca/intermediate_server/newcerts/1009.pem | 32 ++ .../private/azurite.key.pem | 28 + tests/data/ca/intermediate_server/serial | 2 +- tests/data/ca/intermediate_server/serial.old | 2 +- tests/integration/azure/config/compose.yaml | 8 + tests/integration/azure/config/test.yaml | 3 +- .../components/sinks/generated/azure_blob.cue | 155 ++++- .../sinks/generated/azure_logs_ingestion.cue | 83 ++- 28 files changed, 1542 insertions(+), 342 deletions(-) create mode 100644 changelog.d/24729_azure_blob_authentication.feature.md create mode 100644 changelog.d/24729_azure_logs_ingestion.breaking.md create mode 100644 tests/data/ClientCertificateAuth.pfx create mode 100644 tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem create mode 100644 tests/data/ca/intermediate_server/certs/azurite.cert.pem create mode 100644 tests/data/ca/intermediate_server/csr/azurite.csr.pem create mode 100644 tests/data/ca/intermediate_server/newcerts/1009.pem create mode 100644 tests/data/ca/intermediate_server/private/azurite.key.pem diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index a3f2dae3ceea2..b43abdd6fba66 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -210,6 +210,7 @@ greptimedb GSM gvisor gws +gzip'd hadoop Haier Haipad @@ -460,6 +461,7 @@ rumqttc Salesforce Samsung SBT +schema'd scriptblock Sega Segoe diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 60715b9c96c0e..ddd945006f1c4 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -379,6 +379,7 @@ mycie mycorp mydatabase mylabel +mylogstorage mypod myvalue Namazu diff --git a/Cargo.lock b/Cargo.lock index 76e4c912d09ee..45f4a998276fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1668,6 +1668,7 @@ dependencies = [ "async-trait", "azure_core", "futures", + "openssl", "pin-project", "serde", "time", diff --git a/Cargo.toml b/Cargo.toml index 21c40c793a9bf..25ca1f9e0dec9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -302,7 +302,7 @@ aws-smithy-types = { version = "1.2.11", default-features = false, features = [" # Azure azure_core = { version = "0.30", features = ["reqwest", "hmac_openssl"], optional = true } -azure_identity = { version = "0.30", optional = true } +azure_identity = { version = "0.30", features = ["client_certificate"], optional = true } # Azure Storage azure_storage_blob = { version = "0.7", optional = true } @@ -875,8 +875,8 @@ sinks-aws_s3 = ["dep:base64", "dep:md-5", "aws-core", "dep:aws-sdk-s3"] sinks-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sinks-aws_sns = ["aws-core", "dep:aws-sdk-sns"] sinks-axiom = ["sinks-http"] -sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"] -sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity"] +sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:base64"] +sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:base64"] sinks-azure_monitor_logs = [] sinks-blackhole = [] sinks-chronicle = [] @@ -987,7 +987,8 @@ aws-integration-tests = [ ] azure-integration-tests = [ - "azure-blob-integration-tests" + "azure-blob-integration-tests", + "azure-logs-ingestion-integration-tests", ] aws-cloudwatch-logs-integration-tests = ["sinks-aws_cloudwatch_logs"] @@ -1001,6 +1002,7 @@ aws-sqs-integration-tests = ["sinks-aws_sqs"] aws-sns-integration-tests = ["sinks-aws_sns"] axiom-integration-tests = ["sinks-axiom"] azure-blob-integration-tests = ["sinks-azure_blob"] +azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"] chronicle-integration-tests = ["sinks-gcp"] clickhouse-integration-tests = ["sinks-clickhouse"] databend-integration-tests = ["sinks-databend"] diff --git a/changelog.d/24729_azure_blob_authentication.feature.md b/changelog.d/24729_azure_blob_authentication.feature.md new file mode 100644 index 0000000000000..7b224172563b7 --- /dev/null +++ b/changelog.d/24729_azure_blob_authentication.feature.md @@ -0,0 +1,3 @@ +Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. + +authors: jlaundry diff --git a/changelog.d/24729_azure_logs_ingestion.breaking.md b/changelog.d/24729_azure_logs_ingestion.breaking.md new file mode 100644 index 0000000000000..ce0230a352f38 --- /dev/null +++ b/changelog.d/24729_azure_logs_ingestion.breaking.md @@ -0,0 +1,3 @@ +If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` to your sink config (this was previously the default, and now must be explicitly configured). + +authors: jlaundry diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index db0521b4fa137..1311e09edd4f7 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -16,7 +16,8 @@ use crate::{ sinks::{ Healthcheck, VectorSink, azure_common::{ - self, config::AzureBlobRetryLogic, service::AzureBlobService, sink::AzureBlobSink, + self, config::AzureAuthentication, config::AzureBlobRetryLogic, + config::AzureBlobTlsConfig, service::AzureBlobService, sink::AzureBlobSink, }, util::{ BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, @@ -41,6 +42,10 @@ impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct AzureBlobSinkConfig { + #[configurable(derived)] + #[serde(default)] + pub auth: Option, + /// The Azure Blob Storage Account connection string. /// /// Authentication with an access key or shared access signature (SAS) @@ -55,13 +60,33 @@ pub struct AzureBlobSinkConfig { /// | Allowed services | Blob | /// | Allowed resource types | Container & Object | /// | Allowed permissions | Read & Create | + #[configurable(metadata( + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage \ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \ + connection strings secure and not expose them in logs, error messages, or version control systems." + ))] #[configurable(metadata( docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" ))] #[configurable(metadata( docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken" ))] - pub connection_string: SensitiveString, + #[configurable(metadata(docs::examples = "AccountName=mylogstorage"))] + pub connection_string: Option, + + /// The Azure Blob Storage Account name. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "mylogstorage"))] + pub(super) account_name: Option, + + /// The Azure Blob Storage endpoint. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))] + pub(super) blob_endpoint: Option, /// The Azure Blob Storage Account container name. #[configurable(metadata(docs::examples = "my-logs"))] @@ -138,6 +163,10 @@ pub struct AzureBlobSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub tls: Option, } pub fn default_blob_prefix() -> Template { @@ -147,7 +176,10 @@ pub fn default_blob_prefix() -> Template { impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(), + auth: None, + connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()), + account_name: None, + blob_endpoint: None, container_name: String::from("logs"), blob_prefix: default_blob_prefix(), blob_time_format: Some(String::from("%s")), @@ -157,6 +189,7 @@ impl GenerateConfig for AzureBlobSinkConfig { batch: BatchConfig::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: None, }) .unwrap() } @@ -166,11 +199,56 @@ impl GenerateConfig for AzureBlobSinkConfig { #[typetag::serde(name = "azure_blob")] impl SinkConfig for AzureBlobSinkConfig { async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> { + let connection_string: String = match ( + &self.connection_string, + &self.account_name, + &self.blob_endpoint, + ) { + (Some(connstr), None, None) => connstr.inner().into(), + (None, Some(account_name), None) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `account_name`".into(), + ); + } + format!("AccountName={}", account_name) + } + (None, None, Some(blob_endpoint)) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `blob_endpoint`".into(), + ); + } + // BlobEndpoint must always end in a trailing slash + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{}/", blob_endpoint) + }; + format!("BlobEndpoint={}", blob_endpoint) + } + (None, None, None) => { + return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into()); + } + (Some(_), Some(_), _) => { + return Err("Cannot provide both `connection_string` and `account_name`".into()); + } + (Some(_), _, Some(_)) => { + return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); + } + (_, Some(_), Some(_)) => { + return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + } + }; + let client = azure_common::config::build_client( - self.connection_string.clone().into(), + self.auth.clone(), + connection_string.clone(), self.container_name.clone(), cx.proxy(), - )?; + self.tls.clone(), + ) + .await?; let healthcheck = azure_common::config::build_healthcheck( self.container_name.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 63943312fe66d..dbe20a884ff86 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -1,6 +1,8 @@ use std::io::{BufRead, BufReader}; +use std::sync::Arc; use azure_core::http::StatusCode; +use azure_storage_blob::BlobContainerClient; use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; @@ -24,17 +26,24 @@ use crate::{ components::{SINK_TAGS, assert_sink_compliance}, random_events_with_stream, random_lines, random_lines_with_stream, random_string, }, + tls, }; #[tokio::test] async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; + + azure_common::config::build_healthcheck(config.container_name, client) + .expect("Failed to build healthcheck") + .await + .expect("Failed to pass healthcheck"); +} + +#[tokio::test] +async fn azure_blob_healthcheck_passed_with_oauth() { + let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; + let client = config.build_test_client().await; azure_common::config::build_healthcheck(config.container_name, client) .expect("Failed to build healthcheck") @@ -49,12 +58,7 @@ async fn azure_blob_healthcheck_unknown_container() { container_name: String::from("other-container-name"), ..config }; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; assert_eq!( azure_common::config::build_healthcheck(config.container_name, client) @@ -66,10 +70,8 @@ async fn azure_blob_healthcheck_unknown_container() { ); } -#[tokio::test] -async fn azure_blob_insert_lines_into_blob() { +async fn assert_insert_lines_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("lines/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), ..config @@ -88,9 +90,17 @@ async fn azure_blob_insert_lines_into_blob() { } #[tokio::test] -async fn azure_blob_insert_json_into_blob() { +async fn azure_blob_insert_lines_into_blob() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_lines_into_blob_with_oauth() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + +async fn assert_insert_json_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("json/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), encoding: ( @@ -117,6 +127,16 @@ async fn azure_blob_insert_json_into_blob() { assert_eq!(expected, blob_lines); } +#[tokio::test] +async fn azure_blob_insert_json_into_blob() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_json_into_blob_with_oauth() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + #[ignore] #[tokio::test] // This test fails to get the posted blob with "header not found content-length". @@ -177,14 +197,12 @@ async fn azure_blob_insert_json_into_blob_gzip() { assert_eq!(expected, blob_lines); } -#[tokio::test] -async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { +async fn assert_rotate_files_after_the_buffer_size_is_reached(mut config: AzureBlobSinkConfig) { let groups = 3; let (lines, size, input) = random_lines_with_stream_with_group_key(100, 30, groups); let size_per_group = (size / groups) + 10; let blob_prefix = format!("lines-rotate/into/blob/{}", random_string(10)); - let mut config = AzureBlobSinkConfig::new_emulator().await; config.batch.max_bytes = Some(size_per_group); let config = AzureBlobSinkConfig { @@ -211,52 +229,104 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { } } +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { + assert_rotate_files_after_the_buffer_size_is_reached(AzureBlobSinkConfig::new_emulator().await) + .await; +} + +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached_with_oauth() { + assert_rotate_files_after_the_buffer_size_is_reached( + AzureBlobSinkConfig::new_emulator_with_oauth().await, + ) + .await; +} + impl AzureBlobSinkConfig { pub async fn new_emulator() -> AzureBlobSinkConfig { - let address = std::env::var("AZURE_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { - connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into(), - container_name: "logs".to_string(), - blob_prefix: Default::default(), - blob_time_format: None, - blob_append_uuid: None, - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch: Default::default(), - request: TowerRequestConfig::default(), - acknowledgements: Default::default(), - }; + auth: None, + connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: None, + }; config.ensure_container().await; config } - fn to_sink(&self) -> VectorSink { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), + pub async fn new_emulator_with_oauth() -> AzureBlobSinkConfig { + let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let config = AzureBlobSinkConfig { + auth: Some(azure_common::config::AzureAuthentication::MockCredential), + connection_string: Some(format!("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: Some(azure_common::config::AzureBlobTlsConfig { + ca_file: Some(tls::TEST_PEM_CA_PATH.into()), + }), + }; + + config.ensure_container().await; + + config + } + + async fn build_test_client(&self) -> Arc { + azure_common::config::build_client( + self.auth.clone(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + self.tls.clone(), ) - .expect("Failed to create client"); + .await + .expect("Failed to create client") + } + async fn to_sink(&self) -> VectorSink { + let client = self.build_test_client().await; self.build_processor(client).expect("Failed to create sink") } async fn run_assert(&self, input: impl Stream + Send) { // `to_sink` needs to be inside the assertion check - assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().run(input).await }) - .await - .expect("Running sink failed"); + assert_sink_compliance( + &SINK_TAGS, + async move { self.to_sink().await.run(input).await }, + ) + .await + .expect("Running sink failed"); } pub async fn list_blobs(&self, prefix: String) -> Vec { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; // Iterate pager results and collect blob names. Filter by prefix server-side. let mut pager = client @@ -276,12 +346,7 @@ impl AzureBlobSinkConfig { } pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let blob_client = client.blob_client(&blob); @@ -337,12 +402,7 @@ impl AzureBlobSinkConfig { } async fn ensure_container(&self) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let result = client.create_container(None).await; let response = match result { diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index b7f1a9689229f..2dd6aaf69bdd4 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -14,6 +14,8 @@ use super::{config::AzureBlobSinkConfig, request_builder::AzureBlobRequestOption use crate::{ codecs::{Encoder, EncodingConfigWithFraming}, event::{Event, LogEvent}, + sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}, + sinks::prelude::*, sinks::util::{ Compression, request_builder::{EncodeResult, RequestBuilder}, @@ -22,7 +24,10 @@ use crate::{ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { AzureBlobSinkConfig { + auth: Default::default(), connection_string: Default::default(), + account_name: Default::default(), + blob_endpoint: Default::default(), container_name: Default::default(), blob_prefix: Default::default(), blob_time_format: Default::default(), @@ -32,6 +37,7 @@ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { batch: Default::default(), request: Default::default(), acknowledgements: Default::default(), + tls: Default::default(), } } @@ -234,3 +240,317 @@ fn azure_blob_build_request_with_uuid() { assert_eq!(request.content_encoding, None); assert_eq!(request.content_type, "text/plain"); } + +#[tokio::test] +async fn azure_blob_build_config_with_null_auth() { + let config: Result = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + "#, + ); + + match config { + Ok(_) => panic!("Config parsing should have failed due to invalid auth config"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("data did not match any variant of untagged enum"), + "Config parsing did not complain about invalid auth config: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + }) => { + assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); + assert_eq!(azure_client_id, "mock-client-id"); + let secret: String = azure_client_secret.inner().into(); + assert_eq!(secret, "mock-client-secret"); + } + _ => panic!("Expected Specific(ClientSecretCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_certificate_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + certificate_path = "tests/data/ClientCertificateAuth.pfx" + certificate_password = "MockPassword123" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientCertificateCredential { + .. + }) => { + // Expected variant + } + _ => panic!("Expected Specific(ClientCertificateCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name_with_no_auth() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!("Config build should have errored due to missing `auth`"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`auth` configuration must be provided"), + "Config build did not complain about missing `auth`: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_blob_endpoint() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + blob_endpoint = "https://localhost:10000/devstoreaccount1" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_blob_endpoint_with_no_auth() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + blob_endpoint = "https://localhost:10000/devstoreaccount1" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!("Config build should have errored due to missing `auth`"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`auth` configuration must be provided"), + "Config build did not complain about missing `auth`: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!( + "Config build should have errored due to conflicting connection_string and account_name" + ), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`connection_string` and `account_name`"), + "Config build did not complain about conflicting connection_string and account_name: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage;AccountKey=mockkey" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => { + panic!("Config build should have errored due to conflicting Shared Key and Client ID") + } + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str + .contains("Cannot use both Shared Key and another Azure Authentication method"), + "Config build did not complain about conflicting Shared Key and Client ID: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_custom_ca_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [tls] + ca_file = "tests/data/ca/certs/ca.cert.pem" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 139608d9847ab..800e0a6eb3f23 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,19 +1,36 @@ +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; use std::sync::Arc; +use base64::prelude::*; + use azure_core::error::Error as AzureCoreError; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; -use azure_core::http::Url; +use azure_core::http::{ClientMethodOptions, StatusCode, Url}; + +use azure_core::credentials::{TokenCredential, TokenRequestOptions}; +use azure_core::{Error, error::ErrorKind}; + +use azure_identity::{ + AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientCertificateCredential, + ClientCertificateCredentialOptions, ClientSecretCredential, ManagedIdentityCredential, + ManagedIdentityCredentialOptions, UserAssignedId, WorkloadIdentityCredential, + WorkloadIdentityCredentialOptions, +}; + use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; -use azure_core::http::StatusCode; use bytes::Bytes; use futures::FutureExt; use snafu::Snafu; use vector_lib::{ + configurable::configurable_component, json_size::JsonSize, request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, + sensitive_string::SensitiveString, stream::DriverResponse, }; @@ -22,6 +39,374 @@ use crate::{ sinks::{Healthcheck, util::retries::RetryLogic}, }; +/// TLS configuration. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] +pub struct AzureBlobTlsConfig { + /// Absolute path to an additional CA certificate file. + /// + /// The certificate must be in PEM (X.509) format. + #[serde(alias = "ca_path")] + #[configurable(metadata(docs::examples = "/path/to/certificate_authority.crt"))] + #[configurable(metadata(docs::human_name = "CA File Path"))] + pub ca_file: Option, +} + +/// Azure service principal authentication. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, untagged)] +pub enum AzureAuthentication { + #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] + Specific(SpecificAzureCredential), + + /// Mock credential for testing — returns a static fake token + #[cfg(test)] + #[serde(skip)] + MockCredential, +} + +impl Default for AzureAuthentication { + // This should never be actually used. + // This is only needed when using Default::default() (such as unit tests), + // as serde requires `azure_credential_kind` to be specified. + fn default() -> Self { + Self::Specific(SpecificAzureCredential::ManagedIdentity { + user_assigned_managed_identity_id: None, + user_assigned_managed_identity_id_type: None, + }) + } +} + +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +#[derive(Default)] +/// User Assigned Managed Identity Types. +pub enum UserAssignedManagedIdentityIdType { + #[default] + /// Client ID + ClientId, + /// Object ID + ObjectId, + /// Resource ID + ResourceId, +} + +/// Specific Azure credential types. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde( + tag = "azure_credential_kind", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum SpecificAzureCredential { + /// Use Azure CLI credentials + #[cfg(not(target_arch = "wasm32"))] + AzureCli {}, + + /// Use certificate credentials + ClientCertificateCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// PKCS12 certificate with RSA private key. + #[configurable(metadata(docs::examples = "path/to/certificate.pfx"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"))] + certificate_path: PathBuf, + + /// The password for the client certificate, if applicable. + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PASSWORD}"))] + certificate_password: Option, + }, + + /// Use client ID/secret credentials + ClientSecretCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// The [Azure Client Secret][azure_client_secret]. + /// + /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] + azure_client_secret: SensitiveString, + }, + + /// Use Managed Identity credentials + ManagedIdentity { + /// The User Assigned Managed Identity to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + /// or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + }, + + /// Use Managed Identity with Client Assertion credentials + ManagedIdentityClientAssertion { + /// The User Assigned Managed Identity to use for the managed identity. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata( + docs::examples = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-vector/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vector-uami" + ))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + + /// The target Tenant ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_tenant_id: String, + + /// The target Client ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_client_id: String, + }, + + /// Use Workload Identity credentials + WorkloadIdentity { + /// The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID}"))] + tenant_id: Option, + + /// The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID}"))] + client_id: Option, + + /// Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`. + #[configurable(metadata( + docs::examples = "/var/run/secrets/azure/tokens/azure-identity-token" + ))] + #[configurable(metadata(docs::examples = "${AZURE_FEDERATED_TOKEN_FILE}"))] + token_file_path: Option, + }, +} + +#[derive(Debug)] +struct ManagedIdentityClientAssertion { + credential: Arc, + scope: String, +} + +#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] +impl ClientAssertion for ManagedIdentityClientAssertion { + async fn secret(&self, options: Option>) -> azure_core::Result { + Ok(self + .credential + .get_token( + &[&self.scope], + Some(TokenRequestOptions { + method_options: options.unwrap_or_default(), + }), + ) + .await? + .token + .secret() + .to_string()) + } +} + +impl AzureAuthentication { + /// Returns the provider for the credentials based on the authentication mechanism chosen. + pub async fn credential(&self) -> azure_core::Result> { + match self { + Self::Specific(specific) => specific.credential().await, + + #[cfg(test)] + Self::MockCredential => Ok(Arc::new(MockTokenCredential) as Arc), + } + } +} + +impl SpecificAzureCredential { + /// Returns the provider for the credentials based on the specific credential type. + pub async fn credential(&self) -> azure_core::Result> { + let credential: Arc = match self { + #[cfg(not(target_arch = "wasm32"))] + Self::AzureCli {} => AzureCliCredential::new(None)?, + + // requires azure_identity feature 'client_certificate' + Self::ClientCertificateCredential { + azure_tenant_id, + azure_client_id, + certificate_path, + certificate_password, + } => { + let certificate_bytes: Vec = std::fs::read(certificate_path).map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!( + "Failed to read certificate file {}: {e}", + certificate_path.display() + ), + ) + })?; + + // Note: in azure_identity 0.33.0+, this changes to SecretBytes, and the base64 encoding is no longer needed + let certificate_base64: azure_core::credentials::Secret = + BASE64_STANDARD.encode(&certificate_bytes).into(); + + let mut options: ClientCertificateCredentialOptions = + ClientCertificateCredentialOptions::default(); + if let Some(password) = certificate_password { + options.password = Some(password.inner().to_string().into()); + } + + ClientCertificateCredential::new( + azure_tenant_id.clone(), + azure_client_id.clone(), + certificate_base64, + Some(options), + )? + } + + Self::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + if azure_tenant_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_secret.inner().is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + + let secret: String = azure_client_secret.inner().into(); + ClientSecretCredential::new( + &azure_tenant_id.clone(), + azure_client_id.clone(), + secret.into(), + None, + )? + } + + Self::ManagedIdentity { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + ManagedIdentityCredential::new(Some(options))? + } + + Self::ManagedIdentityClientAssertion { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + client_assertion_tenant_id, + client_assertion_client_id, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + let msi: Arc = ManagedIdentityCredential::new(Some(options))?; + let assertion = ManagedIdentityClientAssertion { + credential: msi, + // Future: make this configurable for sovereign clouds? (no way to test...) + scope: "api://AzureADTokenExchange/.default".to_string(), + }; + + ClientAssertionCredential::new( + client_assertion_tenant_id.clone(), + client_assertion_client_id.clone(), + assertion, + None, + )? + } + + Self::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + } => { + let options = WorkloadIdentityCredentialOptions { + tenant_id: tenant_id.clone(), + client_id: client_id.clone(), + token_file_path: token_file_path.clone(), + ..Default::default() + }; + + WorkloadIdentityCredential::new(Some(options))? + } + }; + Ok(credential) + } +} + #[derive(Debug, Clone)] pub struct AzureBlobRequest { pub blob_data: Bytes, @@ -126,10 +511,12 @@ pub fn build_healthcheck( Ok(healthcheck.boxed()) } -pub fn build_client( +pub async fn build_client( + auth: Option, connection_string: String, container_name: String, proxy: &crate::config::ProxyConfig, + tls: Option, ) -> crate::Result> { // Parse connection string without legacy SDK let parsed = ParsedConnectionString::parse(&connection_string) @@ -140,16 +527,26 @@ pub fn build_client( .map_err(|e| format!("Failed to build container URL: {e}"))?; let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; + let mut credential: Option> = None; + // Prepare options; attach Shared Key policy if needed let mut options = BlobContainerClientOptions::default(); - match parsed.auth() { - Auth::Sas { .. } | Auth::None => { - // No extra policy; SAS is in the URL already (or anonymous) + match (parsed.auth(), &auth) { + (Auth::None, None) => { + warn!("No authentication method provided, requests will be anonymous."); + } + (Auth::Sas { .. }, None) => { + info!("Using SAS token authentication."); } - Auth::SharedKey { - account_name, - account_key, - } => { + ( + Auth::SharedKey { + account_name, + account_key, + }, + None, + ) => { + info!("Using Shared Key authentication."); + let policy = SharedKeyAuthorizationPolicy::new( account_name, account_key, @@ -162,6 +559,41 @@ pub fn build_client( .per_call_policies .push(Arc::new(policy)); } + (Auth::None, Some(AzureAuthentication::Specific(..))) => { + info!("Using Azure Authentication method."); + let credential_result: Arc = + auth.unwrap().credential().await.map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!("Failed to configure Azure Authentication: {e}"), + ) + })?; + credential = Some(credential_result); + } + (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both SAS token and another Azure Authentication method at the same time", + ))); + } + (Auth::SharedKey { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both Shared Key and another Azure Authentication method at the same time", + ))); + } + #[cfg(test)] + (Auth::None, Some(AzureAuthentication::MockCredential)) => { + warn!("Using mock token credential authentication."); + credential = Some(auth.unwrap().credential().await.unwrap()); + } + #[cfg(test)] + (_, Some(AzureAuthentication::MockCredential)) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both connection string auth and mock credential at the same time", + ))); + } } // Use reqwest v0.12 since Azure SDK only implements HttpClient for reqwest::Client v0.12 @@ -191,12 +623,90 @@ pub fn build_client( reqwest_builder = reqwest_builder.proxy(p); } } + + if let Some(AzureBlobTlsConfig { ca_file }) = &tls + && let Some(ca_file) = ca_file + { + let mut buf = Vec::new(); + File::open(ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; + + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( reqwest_builder .build() .map_err(|e| format!("Failed to build reqwest client: {e}"))?, ))); - let client = - BlobContainerClient::from_url(url, None, Some(options)).map_err(|e| format!("{e}"))?; + let client = BlobContainerClient::from_url(url, credential, Some(options)) + .map_err(|e| format!("{e}"))?; Ok(Arc::new(client)) } + +#[cfg(test)] +#[derive(Debug)] +struct MockTokenCredential; + +#[cfg(test)] +#[async_trait::async_trait] +impl TokenCredential for MockTokenCredential { + async fn get_token( + &self, + scopes: &[&str], + _options: Option>, + ) -> azure_core::Result { + let Some(scope) = scopes.first() else { + return Err(Error::with_message( + ErrorKind::Credential, + "no scopes were provided", + )); + }; + + // serde_json sometimes does and sometimes doesn't preserve order, be careful to sort + // the claims in alphabetical order to ensure a consistent base64 encoding for testing + let jwt = serde_json::json!({ + "aud": scope.strip_suffix("/.default").unwrap_or(*scope), + "exp": 2147483647, + "iat": 0, + "iss": "https://sts.windows.net/", + "nbf": 0, + }); + + // JWTs do not include standard base64 padding. + // this seemed cleaner than importing a new crates just for this function + let jwt_base64 = format!( + "e30.{}.", + BASE64_STANDARD + .encode(serde_json::to_string(&jwt).unwrap()) + .trim_end_matches("=") + ) + .to_string(); + + warn!( + "Using mock token credential, JWT: {}, base64: {}", + serde_json::to_string(&jwt).unwrap(), + jwt_base64 + ); + + Ok(azure_core::credentials::AccessToken::new( + jwt_base64, + azure_core::time::OffsetDateTime::now_utc() + std::time::Duration::from_secs(3600), + )) + } +} + +#[cfg(test)] +#[tokio::test] +async fn azure_mock_token_credential_test() { + let credential = MockTokenCredential; + let access_token = credential + .get_token(&["https://example.com/.default"], None) + .await + .expect("valid credential should return a token"); + assert_eq!( + access_token.token.secret(), + "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMTQ3NDgzNjQ3LCJpYXQiOjAsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." + ); +} diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 4e876a18a26bd..3b105d549d0c1 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -1,20 +1,14 @@ use std::sync::Arc; -use azure_core::credentials::{TokenCredential, TokenRequestOptions}; -use azure_core::http::ClientMethodOptions; -use azure_core::{Error, error::ErrorKind}; - -use azure_identity::{ - AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, - ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, - WorkloadIdentityCredential, -}; -use vector_lib::{configurable::configurable_component, schema, sensitive_string::SensitiveString}; +use azure_core::credentials::TokenCredential; + +use vector_lib::{configurable::configurable_component, schema}; use vrl::value::Kind; use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ + azure_common::config::AzureAuthentication, prelude::*, util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, }, @@ -65,7 +59,6 @@ pub struct AzureLogsIngestionConfig { pub stream_name: String, #[configurable(derived)] - #[serde(default)] pub auth: AzureAuthentication, /// [Token scope][token_scope] for dedicated Azure regions. @@ -129,191 +122,6 @@ impl Default for AzureLogsIngestionConfig { } } -/// Configuration of the authentication strategy for interacting with Azure services. -#[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] -#[serde(deny_unknown_fields, untagged)] -pub enum AzureAuthentication { - /// Use client credentials - #[derivative(Default)] - ClientSecretCredential { - /// The [Azure Tenant ID][azure_tenant_id]. - /// - /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_tenant_id: String, - - /// The [Azure Client ID][azure_client_id]. - /// - /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_client_id: String, - - /// The [Azure Client Secret][azure_client_secret]. - /// - /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] - azure_client_secret: SensitiveString, - }, - - /// Use credentials from environment variables - #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] - Specific(SpecificAzureCredential), -} - -/// Specific Azure credential types. -#[configurable_component] -#[derive(Clone, Debug, Eq, PartialEq)] -#[serde( - tag = "azure_credential_kind", - rename_all = "snake_case", - deny_unknown_fields -)] -pub enum SpecificAzureCredential { - /// Use Azure CLI credentials - #[cfg(not(target_arch = "wasm32"))] - AzureCli {}, - - /// Use Managed Identity credentials - ManagedIdentity { - /// The User Assigned Managed Identity (Client ID) to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - }, - - /// Use Managed Identity with Client Assertion credentials - ManagedIdentityClientAssertion { - /// The User Assigned Managed Identity (Client ID) to use for the managed identity. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - - /// The target Tenant ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_tenant_id: String, - - /// The target Client ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_client_id: String, - }, - - /// Use Workload Identity credentials - WorkloadIdentity {}, -} - -#[derive(Debug)] -struct ManagedIdentityClientAssertion { - credential: Arc, - scope: String, -} - -#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] -impl ClientAssertion for ManagedIdentityClientAssertion { - async fn secret(&self, options: Option>) -> azure_core::Result { - Ok(self - .credential - .get_token( - &[&self.scope], - Some(TokenRequestOptions { - method_options: options.unwrap_or_default(), - }), - ) - .await? - .token - .secret() - .to_string()) - } -} - -impl AzureAuthentication { - /// Returns the provider for the credentials based on the authentication mechanism chosen. - pub async fn credential(&self) -> azure_core::Result> { - match self { - Self::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - if azure_tenant_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_secret.inner().is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - let secret: String = azure_client_secret.inner().into(); - let credential: Arc = ClientSecretCredential::new( - &azure_tenant_id.clone(), - azure_client_id.clone(), - secret.into(), - None, - )?; - Ok(credential) - } - - Self::Specific(specific) => specific.credential().await, - } - } -} - -impl SpecificAzureCredential { - /// Returns the provider for the credentials based on the specific credential type. - pub async fn credential(&self) -> azure_core::Result> { - let credential: Arc = match self { - #[cfg(not(target_arch = "wasm32"))] - Self::AzureCli {} => AzureCliCredential::new(None)?, - - Self::ManagedIdentity { - user_assigned_managed_identity_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - ManagedIdentityCredential::new(Some(options))? - } - - Self::ManagedIdentityClientAssertion { - user_assigned_managed_identity_id, - client_assertion_tenant_id, - client_assertion_client_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - let msi: Arc = ManagedIdentityCredential::new(Some(options))?; - let assertion = ManagedIdentityClientAssertion { - credential: msi, - // Future: make this configurable for sovereign clouds? (no way to test...) - scope: "api://AzureADTokenExchange/.default".to_string(), - }; - - ClientAssertionCredential::new( - client_assertion_tenant_id.clone(), - client_assertion_client_id.clone(), - assertion, - None, - )? - } - - Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, - }; - Ok(credential) - } -} - impl AzureLogsIngestionConfig { #[allow(clippy::too_many_arguments)] pub(super) async fn build_inner( diff --git a/src/sinks/azure_logs_ingestion/tests.rs b/src/sinks/azure_logs_ingestion/tests.rs index 26eda91338836..46095071bd28c 100644 --- a/src/sinks/azure_logs_ingestion/tests.rs +++ b/src/sinks/azure_logs_ingestion/tests.rs @@ -8,6 +8,8 @@ use vector_lib::config::log_schema; use azure_core::credentials::{AccessToken, TokenCredential}; use azure_core::time::OffsetDateTime; +use crate::sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}; + use super::config::AzureLogsIngestionConfig; use crate::{ @@ -26,50 +28,22 @@ fn generate_config() { #[tokio::test] async fn basic_config_error_with_no_auth() { - let config: AzureLogsIngestionConfig = toml::from_str::( - r#" + let config: Result = + toml::from_str::( + r#" endpoint = "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" dcr_immutable_id = "dcr-00000000000000000000000000000000" stream_name = "Custom-UnitTest" "#, - ) - .expect("Config parsing failed"); - - assert_eq!( - config.endpoint, - "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" - ); - assert_eq!( - config.dcr_immutable_id, - "dcr-00000000000000000000000000000000" - ); - assert_eq!(config.stream_name, "Custom-UnitTest"); - assert_eq!(config.token_scope, "https://monitor.azure.com/.default"); - assert_eq!(config.timestamp_field, "TimeGenerated"); - - match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - assert_eq!(azure_tenant_id, ""); - assert_eq!(azure_client_id, ""); - let secret: String = azure_client_secret.inner().into(); - assert_eq!(secret, ""); - } - _ => panic!("Expected ClientSecretCredential variant"), - } + ); - let cx = SinkContext::default(); - let sink = config.build(cx).await; - match sink { - Ok(_) => panic!("Config build should have errored due to missing auth info"), + match config { + Ok(_) => panic!("Config parsing should have failed due to missing auth config"), Err(e) => { let err_str = e.to_string(); assert!( - err_str.contains("`auth.azure_tenant_id` is blank"), - "Config build did not complain about azure_tenant_id being blank: {}", + err_str.contains("missing field `auth`"), + "Config parsing did not complain about missing auth field: {}", err_str ); } @@ -85,6 +59,7 @@ fn basic_config_with_client_credentials() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -105,17 +80,17 @@ fn basic_config_with_client_credentials() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, - } => { + }) => { assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); assert_eq!(azure_client_id, "mock-client-id"); let secret: String = azure_client_secret.inner().into(); assert_eq!(secret, "mock-client-secret"); } - _ => panic!("Expected ClientSecretCredential variant"), + _ => panic!("Expected Specific(ClientSecretCredential) variant"), } } @@ -146,11 +121,7 @@ fn basic_config_with_managed_identity() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::Specific( - crate::sinks::azure_logs_ingestion::config::SpecificAzureCredential::ManagedIdentity { - .. - }, - ) => { + AzureAuthentication::Specific(SpecificAzureCredential::ManagedIdentity { .. }) => { // Expected variant } _ => panic!("Expected Specific(ManagedIdentity) variant"), @@ -182,6 +153,7 @@ async fn correct_request() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -292,6 +264,7 @@ async fn mock_healthcheck_with_400_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -361,6 +334,7 @@ async fn mock_healthcheck_with_403_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index 3a9c3c049a0fa..e5584de61dd02 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -26,7 +26,7 @@ pub mod aws_s_s; pub mod axiom; #[cfg(feature = "sinks-azure_blob")] pub mod azure_blob; -#[cfg(feature = "sinks-azure_blob")] +#[cfg(any(feature = "sinks-azure_blob", feature = "sinks-azure_logs_ingestion",))] pub mod azure_common; #[cfg(feature = "sinks-azure_logs_ingestion")] pub mod azure_logs_ingestion; diff --git a/tests/data/ClientCertificateAuth.pfx b/tests/data/ClientCertificateAuth.pfx new file mode 100644 index 0000000000000000000000000000000000000000..e289b55375a4ac01132352e54813ffd76405bcc0 GIT binary patch literal 2467 zcmai$X*3iH8^_JYjO|(m*)GNs8rRtOeN7nIn_)7@zGSB`jBHaP(Y2&TlOjuK24hcl zDceP|zPhL+GMI$>y61gQ_v8KWoaa3M|9Q^y>GwYmiU@lM1b|RPmgra;t>(+DF#`Fr0R=8`-8d_*|TW-N^ zbl*q?#EcG5mTh$AF9my5!#w>BN%mvzvYCMHjMl9IEW7mY*aR|^%vM|mk z>D$T_eLDX7@~?mtF+1NH>3DzL7G&!OEmFHc3r%yrn8ag zH8OA8F}bG=p>N?#M!pyvyu4cQ#Qn?L7u)AQZ+B7!Q>u$)TiA@enyp?*wo7RZbgbNr z&9_oG>09(IIZv_)5y+kBwFf*c{#`)C3Juaoj0>HebqYigAvb>V_czdx7%&kMaVm$N zRv;_K|5@i`2LMk^l~Yma-vGXH&|ttURI>b6fZ?ozpeNh!`Rl5i<{r?J`@ux;7!I5> zk~oIIy(wTih5{n}gchSthP}|EWtvrAb1wwuF6w4`MtP(6ZzlOy;D-whH6j*woj-bk z)v4+nR`TDo8;<5`qfGI)l#GY2W8vLL+qKq31G%0>Uz1m#-SPChsR%41Y%^EtCv>o!YV;iZjxy@}h{{#bOtDMheWo zw(4F>xS3U(D%6LzC{1e0{q;x_u^s9p{A(?5C~05GqJXvpEEI}q(C|}I>7$HpPTa;( zrebEj=W6qY%t~uLkEblEA|;}Cy=kE5_?bzyuZr-)^J(I+G-#qWHO=2a1)KPk4-x38 z>=FAgP}b@({qg}lYNQf-c}`5X>Fr+!zr!Jbn*O(J*30vA{65;tEAe7zed!atQ<2X3 z#17Kh&D-g=(PI1y>@p-1C(HK!inEb}_oaA>{NkD#QW%M@eom+@KXV%@A#!?&T}rD< zd0yKg;gPnNgH3^^q=a#VR}ih@({t}!#?1yiEy;M0UXBJT9$34IMNi(SJ;TYLnr+tk z*c(E97tZ-w>DIX5PPWM^{Zd9y^_Z4iD?Fk7*@xJ;EWY?T^6!?b!bNqx<> zC%b<-DHuB|936ZiiCucg_3C?4^B>+chJ0pT;DP3e+L7ry8H^|a7^7h7O5T{@p4eRK zdXJ3Kb?FhAD+sCCqY7rE5>n&Kc7x_9>@w+kq)nE)XnBIt9mRn{-n-@f!Rwo%8sAp; z`G7ryL63GLEZ?Q$bVhwSc+|UTM%u5xBI7fmr&&j7eD#MYwMWopHu>PmHzWgMvdW7! zTeBA{-vzd?b+Y#GBWkjPq4owL1{?ak(~%HdqsX(sVK_!hiGkb$FvLH8pi|->j?aRt zs+J{|Z@0&nhxZgax?n>9Zo{>iq`l6pl zESYILy?h-=+#aO1HPU*Di&}%zsdiGF+oKSstNb?mjUSkxHeCO+WGgXJK4Rzdob#%S zo?N4OC+z_dCiFoKYwe?Ek-1%UmYwfp+*QKsb%~(F&Z`K1u8>Sg)r)PTx3td8i&(1F zvzK4x@G5O7&%dzVRJmnWgVdFg)8wg8DMaeWaeWut0Qw&nV@ZrO%V$}4UIx81Gk@E* z;39mM#(E!rt_hFo7t+-=p4R^`KKh%MSVeS38RG9F! zYl0t@52PjH7ZfO3M~=Hg4Pu^`L)OV>4BM%Ed!SMfMzq&s|D?1q`Z!vw|a*UpR2qV1*M+9 zLztw&9}G=BTDO3B;VLfa6)@9bE?Qz51B!p-smP+Fe+Dc}Kt&J?NABjzJC%MW3RI`X za2d$ig=@Blp-fOJD6XIH4G;if0ZaN8uILs-Ob3Hud6PjPmwKTu_?mrd<`jRwu)MFc Tg&i2m^(Z6qB2fO!-% ca/intermediate_server/certs/localhost-chain.cert.pem +ca/intermediate_server/private/azurite.key.pem: + openssl genrsa -out ca/intermediate_server/private/azurite.key.pem 2048 + +ca/intermediate_server/csr/azurite.csr.pem: ca/intermediate_server/private/azurite.key.pem + openssl req -config ca/intermediate_server/openssl.cnf \ + -key ca/intermediate_server/private/azurite.key.pem \ + -subj '/CN=azurite/OU=Vector/O=Datadog/ST=New York/L=New York/C=US' \ + -addext 'subjectAltName=DNS:azurite' \ + -addext 'basicConstraints=critical,CA:FALSE' \ + -addext 'extendedKeyUsage=serverAuth' \ + -new -sha256 -out ca/intermediate_server/csr/azurite.csr.pem + +ca/intermediate_server/certs/azurite.cert.pem: ca/intermediate_server/csr/azurite.csr.pem + openssl ca -batch -config ca/intermediate_server/openssl.cnf \ + -extensions server_cert -days 3650 -notext -md sha256 \ + -in ca/intermediate_server/csr/azurite.csr.pem \ + -out ca/intermediate_server/certs/azurite.cert.pem + +ca/intermediate_server/certs/azurite-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/azurite.cert.pem + cat ca/intermediate_server/certs/azurite.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/azurite-chain.cert.pem + ca/intermediate_server/private/elasticsearch-secure.key.pem: openssl genrsa -out ca/intermediate_server/private/elasticsearch-secure.key.pem 2048 diff --git a/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem new file mode 100644 index 0000000000000..f646a0fe3c20a --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj +dG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNV +BAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMwHhcN +MjIwNjA3MjIyNzUzWhcNMzIwNjA0MjIyNzUzWjBrMQswCQYDVQQGEwJVUzERMA8G +A1UECAwITmV3IFlvcmsxEDAOBgNVBAoMB0RhdGFkb2cxDzANBgNVBAsMBlZlY3Rv +cjEmMCQGA1UEAwwdVmVjdG9yIEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCy/mB0/ZwfgKrSZPQIFaGPtRA9xL2N +o2SsHndZ8k2XOCV225Hb2fzNH+o2WGNSjwmGjLP/uXb47KH0cHCAyFGzSjp++8/O +zoZaFiO0P5El02hQxmoabO3Cqu/N62EFsLfpSM828JM6YOn9p+WXUDn1+YPNoOOE +H142p4/RjFnXNHkzR3geXU4Pfi3KXDrMi8vK42lDqXPLPs6rhreBAfQ2dsYyqhz6 +tg6FzZuXxxzEYyYtNgGh+zTji99WCBMLbCmRcDurRjdTDO7m4O3PrwbGUy0xdLeb +HJiNGvUDCPH4bfwLiNqwVIZY38RBCAqbCnrqRhDaZIfAUev4mq3Kqh6KUeO/U7Vx +/5J5rL5ApREKOfWPATHMprBuEU2rs3N+MPBA04HoiFlu311urCxVEA1qsZCTkoCg +GHuDIVSU4E4hT4co95/J0to4zWgPlfPg1+cXyU8lAIMe7JdCGkG9cDe7Umw/GSbt +ZdoCMQZ6WyyiW2Hw+7sFD3V3VzYa5YA/rjKZRduPmGWKrs+mAa5J5pM2M22rrjbd +EpfTHWLS9s6cPN3/jxpCxn6Hv/KhIYRAcIterugag1+clvS1ajVjxBRavOxPBsf+ +hYdh7S5NTZnT98gjkc3yOuGQm7BPtXau+IYZRlWcB0dJ4/E2P69hmWQezSo9VVWh +5/K1RkbPvqTGZQIDAQABo2YwZDAdBgNVHQ4EFgQUPD06L8zVggN9mcRY8eHbNu+t +DUEwHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wEgYDVR0TAQH/BAgw +BgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAGqaGBuL +2J6Na8RHx/GmSeuZFiVcWhmd/I9bVpeMSYHSZujA2nay6OGaUYs0Lq/G5OKgsuT9 +AIHnsn7VUE1zqoDfXac/K8sXlOig8if7rTb+06jgymaP1YSELg3R+pBsdkZnXVil +izh/9FvzoyV+QQlIhojqCIybVFgxa1XFHq4QCPhDfwkg+tp9RctfwNmWgsJ63H19 +RmxN+H2xIrySvObwXnB4j6D4wvgu468QXQMEuSsnLcIQFg6Zteqe8fixbqTiOTBf +Dk1k+EpB9VMEkIPvMdfa48vseXdBEe6Ma9zGuJC76q4q1ZapVLTvOUP5Y24khlgd +cj5tfP7o7yc6HqymfXAcD1lzP2JQhqaRxA4I18Nrd+aHi+G1EM2c3cicuD3n6Iw9 +9oqdCwmMfS25fv5cyA5B6hRusIZ9wRopTi7at+JHl0GIt/FelaTYI7kRmAqgakQe +oEKLpXcH8lRJW802DmXm7ka4eQzwxa7Ngyf8O+JOFtGO0+EshuLJovxiPl6IyLyG +NJ/dHq3ad+46YVManbHdyjHxgT5PSvJFkq0Yluvf44NIyP5QRTCAvfH76bu7hXgS +QoQj5t5ILn6meQRTR79r2iwpQTanPLTEdoZvmrE4TeUBev9BA5KpiPPA3i3ZF/oV +0EYorXCNri7M/jylGW7AuWvNUyaVR6xgxAn6 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJAKhPL9BkNaFGMA0GCSqGSIb3DQEBCwUAMGoxEjAQBgNV +BAMMCVZlY3RvciBDQTEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9n +MREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxCzAJBgNVBAYT +AlVTMB4XDTIyMDYwNzIyMjc1MloXDTQyMDYwMjIyMjc1MlowajESMBAGA1UEAwwJ +VmVjdG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9c1T+NXTNmqiiV36NSEJt +7mo0cyv8Byk2ZGdC85vHBm45QDY5USoh0vgonzPpWgSMggPn1WbR0f1y+LBwXdlM ++ZyZh2RVVeUrSjJ88lLHVn4DfywpdDkwQaFj1VmOsj2I9rMMrgc5x5n1Hj7lwZ+t +uPVSAGmgKp4iFfzLph9r/rjP1TUAnVUComfTUVS+Gd7zoGPOc14cMJXG6g2P2aAU +P6dg5uQlTxRmagnlx7bwm3lRwv6LMtnAdnjwBDBxr933nucAnk21GgE92GejiO3Z +OwlzIdzBI23lPcWi5pq+vCTgAArNq24W1Ha+7Jn5QewNTGKFyyYAJetZAwCUR8QS +Ip++2GE2pNhaGqcV5u1Tbwl02eD6p2qRqjfgLxmb+aC6xfl0n9kiFGPZppjCqDEW +sw+gX66nf+qxZVRWpJon2kWcFvhTnLqoa3T3+9+KIeamz2lW6wxMnki/Co2EA1Wa +mmedaUUcRPCgMx9aCktRkMyH6bEY8/vfJ07juxUsszOc46T00Scmn6Vkuo9Uc3Kf +2Q2N6Wo4jtyAiMO4gAwq5kzzpBAhNgRfLHOb83r2gAUj2Y4Vln/UUR/KR8ZbJi4i +r1BjX16Lz3yblJXXb1lp4uZynlbHNaAevXyGlRqHddM2ykKtAX/vgJcZRGSvms11 +uce/cqzrzx60AhpLRma5CwIDAQABo2MwYTAdBgNVHQ4EFgQURTWK6ARqnZkz8rkt +Uc5PrtasIh8wHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAEf5 +TR3hq/DtSAmsYotu1lAWz/OlTpG+7AdqSOHB878X4ETN3xaQ+KWvSwvf0K70ZDTV +tFOTh/r43cpzPifPKd1P+2ctnQEzrBtAacvyETLq1ABRK9VJOtfJ6Xk5KZXPhKdY +t353PQgBgW8YzQ2adq2B7FtgIlX7f1DIndjcMZBbolETR6xt9QwB/UnPI7Mwt01T ++bCBhr1fWAbZ4YAMlQ0xRam4qUOTjxgfmePrmSrv4HO7cXHMsRMLiXk+BLcx959/ +K/B6xzpzn6366Eqnqlo/uDiMpo5ud2I/Snz5PduB6oLztPMEf/8RmkG5tpHXYdWr +tM64WqNGO+ikluIrrtYvtyZS4DfsLAMfMYZcxX/Uw56gHo0i2c8I6+6JvGWdvOJ0 +FjrsKeIQoRlV77z025kI4V9jKi3XNMEsAIH+W7KNSut0X80yX7SugvQGoe0GDkXu +0fy8hMC3uTN2LEycYFRRfoIeKPLi6OZFK0PdS2E15d8PEU3n3W4eBCPgMtmiOKLY +d8QNBC8XLAuBoK9R8luCJpOJWUcFXjLpjcDab4V2hKTuAs+GQyDh/Xx4wF1yHX0r +zIkyN0EkOD/SvD8X4uFaM4mdsAh+ucn4ryUV7i5PgvDM9z4InHAMAee1ebBl0U+h ++NzMWF5c5OwxD5o6/Wh1HopmzJiVNT2v9u0kHT/f +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/certs/azurite.cert.pem b/tests/data/ca/intermediate_server/certs/azurite.cert.pem new file mode 100644 index 0000000000000..3e2f680ef37c0 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/csr/azurite.csr.pem b/tests/data/ca/intermediate_server/csr/azurite.csr.pem new file mode 100644 index 0000000000000..09cf2dd0bc3f7 --- /dev/null +++ b/tests/data/ca/intermediate_server/csr/azurite.csr.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC9TCCAd0CAQAwaDEQMA4GA1UEAwwHYXp1cml0ZTEPMA0GA1UECwwGVmVjdG9y +MRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwI +TmV3IFlvcmsxCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVPbGW0LqD1WO42dSmUMQ2iRGsF +eHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bRRInmuyocDTvm4t7VU+ybnPD7 +MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3QZ4loS61BR5TxrCdnpE++gxh +7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ63PaXXvGpGiDwPqUqi0WhIYp +73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFnVr8lMVavcDcoTrHdJ8PXN4Ei +pgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABoEgwRgYJKoZIhvcNAQkOMTkwNzAS +BgNVHREECzAJggdhenVyaXRlMAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwDQYJKoZIhvcNAQELBQADggEBAIbILz+IbvX9126CRLqmtlp4s1q2L+A2 +jYz+qk1G1UhImumfEcLMBfj6X/ZQ5IRIrH8BZZtW9ungXUS/9eYKUCQP52GXPPZJ +WkS3sERjvktEP7orNrTLo7v36f9P0J9eMjN8XYhcqHmZB5OyoQ4S3ndqgE4+HV8j +OyP9lZC1Urjrre/S7pyxxnkVftxXtVHXzB0MgoQa77U/ukqeaMYIw3qWRqGwwl44 +JnZm05mukTpYnDVmo+J2Ra1uNOJf9/SIEnawyL1ROYCINxgIcdodrpuxz+dnYT0K +zvsbMGQ1MTacJK61Pw0EBbxERB9ZhWnApRdeuhrF5oq9uGEzD/76wTg= +-----END CERTIFICATE REQUEST----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index 0402ad0946130..b2a0481603b78 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -6,3 +6,4 @@ V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar +V 360301205253Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite diff --git a/tests/data/ca/intermediate_server/index.txt.old b/tests/data/ca/intermediate_server/index.txt.old index ab4efe2c2f6bd..0402ad0946130 100644 --- a/tests/data/ca/intermediate_server/index.txt.old +++ b/tests/data/ca/intermediate_server/index.txt.old @@ -5,3 +5,4 @@ V 320613195026Z 1004 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=kafka V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq +V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar diff --git a/tests/data/ca/intermediate_server/newcerts/1009.pem b/tests/data/ca/intermediate_server/newcerts/1009.pem new file mode 100644 index 0000000000000..3e2f680ef37c0 --- /dev/null +++ b/tests/data/ca/intermediate_server/newcerts/1009.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/private/azurite.key.pem b/tests/data/ca/intermediate_server/private/azurite.key.pem new file mode 100644 index 0000000000000..545d8904a3d73 --- /dev/null +++ b/tests/data/ca/intermediate_server/private/azurite.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC+cDIKkK8OivJI +I1qy1Vb3Hf7fVNrhJU9sZbQuoPVY7jZ1KZQxDaJEawV4e8ziC5euFeWgFV/fkvU7 +JZGn9uW0eWV4uuvnxtFEiea7KhwNO+bi3tVT7Juc8PsyF02xsyajYcGfXZwRjs+z +PFU7BxnyPRc4e0j697dBniWhLrUFHlPGsJ2ekT76DGHsqFQbXJOIqYQS3nzu9Ei5 +UFY2ssVWFhA3qZz7Yknrc9pde8akaIPA+pSqLRaEhinvddXIipseK8jbtOK6/+f2 +Bwc8C3ZTBVu/UCqoUWdWvyUxVq9wNyhOsd0nw9c3gSKmC+ywkXXiAUWDsvRTgA6P +jG7fV76JAgMBAAECggEAEXeC73sjw0a1QC6V9A8jQdkrdlp3FO1yInQVma1Ds5tt +vUNKB1HDz1itkMZyHU2I5Pu3Kv6q43u8KGeiu3Am023LA87JMmIG8a1gT0xmdERJ +QgfEM0VhZHyp3YdLpf/TjGq78p0IYofhvwPKoXZeR9yYk6KjJ/mugkM6GlWJXuWn +sE2ZYL4Wl6d//1/9D6xWfNVU1zCH7ScbFFvQKkBxURLpnQlgkfhr8HMrCQyV4Lps +6n166zcQKJO4mwGrWyu75HU8vcBYv1+6WbJ1Rgke6lL3dzjIhzE7C61i3dI636Aq +l0wZ3NVeysyrn5Xzu5vfq0D9/pt+Of1PE0FmmlRogQKBgQDhaY8OyUafMYhGnike +7K1ho60tFHE5vgxMpoUTinLyARgW8bA9VRtQ++PW90Z1sHzXAHjb30YZcYEuKaCW +SvIWg49/L4bORX2jY4UPZvESemJtnIXZkuuiGoYaXjIwNudorT1siCD/GYM24Uko +ax0QbFsNRtq89QE/ibKdFLRfUQKBgQDYR7JfauKmsY6OQjB8RvFe50AnnPfiJSZ+ +rD7H7gXWlt3q9zpsU3xBK+fwI9QsvTlUNsIkFhJpu6znCzK26nLIJCqGcm0W+9Ah +rJR+0TXyafQn1d1dTrtn7XVIL6QQEuUY9jggsAXWBejdPCeaVxplEiJa1i9x3YOp +OQWmMTvNuQKBgCiGF6fq231nJD691Fqw5gK1sD54fFqLJh7pmOcIbt2/AJuvW6XL +FRwcDLvqvIoP7oGgnhm5LBsK4tRvu2UJmDgf8r5ExxFyQMIM9DDuqsxNoEBgcVfK +J/5+kjlPUeqFFFknO/G1D2mNJp/JJKPVjeYT9NKQOGbcDRtlH+1JeZvhAoGAZkOg +Z9WWTdNu4H0Th+/TeVhG0XQ7EUcXqJWxKb+2Kv0y+ULk8QuYmQg1pyqJzI28acFq +kr2M/0mqO6Tj2fGJTHEtWl0Ij/GJPCLqI/ywUWsf8yYAgXoUytNQvU0peiA1C1SA +vZP9bnFk5hbncub0qA2nCOR1kpV3B7DapvZonKECgYEAjzpfOhaULZgwnkYOkoFw +VS+tDOVNVfjDOjuV82B/0zCgCW8dv2EeTCobEkIbSl8GYMzukNZQR8r3ZAxbs1By +Ieliad2FD/s0MtsPiGcxJN4twbDQO/SvLFtjQ+jCysgOVON6KiqpGVRcEL+6U4ZD +NiqT36xmXpmkZ05bMxOXPLM= +-----END PRIVATE KEY----- diff --git a/tests/data/ca/intermediate_server/serial b/tests/data/ca/intermediate_server/serial index 6cb3869343bf8..4e75b247d2247 100644 --- a/tests/data/ca/intermediate_server/serial +++ b/tests/data/ca/intermediate_server/serial @@ -1 +1 @@ -1009 +100A diff --git a/tests/data/ca/intermediate_server/serial.old b/tests/data/ca/intermediate_server/serial.old index 617ba1c154075..6cb3869343bf8 100644 --- a/tests/data/ca/intermediate_server/serial.old +++ b/tests/data/ca/intermediate_server/serial.old @@ -1 +1 @@ -1008 +1009 diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index fe3541bf2a07c..384a65ab0e42d 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -7,6 +7,14 @@ services: volumes: - /var/run:/var/run + azurite: + image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} + command: azurite --blobHost 0.0.0.0 --blobPort 14430 --queuePort 14431 --tablePort 14432 --loose --oauth basic --cert /certs/azurite-chain.cert.pem --key /private/azurite.key.pem + volumes: + - /var/run:/var/run + - ../../../data/ca/intermediate_server/certs:/certs + - ../../../data/ca/intermediate_server/private/azurite.key.pem:/private/azurite.key.pem + networks: default: name: ${VECTOR_NETWORK} diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index da00ceed79ad4..1b3a0e4cbe151 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -4,7 +4,8 @@ features: test_filter: ::azure_ env: - AZURE_ADDRESS: local-azure-blob + AZURITE_ADDRESS: local-azure-blob + AZURITE_OAUTH_ADDRESS: azurite HEARTBEAT_ADDRESS: 0.0.0.0:8080 LOGSTASH_ADDRESS: 0.0.0.0:8081 diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 5ea0c1dd20221..8d962c30adbdb 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -1,6 +1,16 @@ package metadata generated: components: sinks: azure_blob: configuration: { + account_name: { + description: """ + The Azure Blob Storage Account name. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["mylogstorage"] + } acknowledgements: { description: """ Controls how acknowledgements are handled for this sink. @@ -27,6 +37,123 @@ generated: components: sinks: azure_blob: configuration: { type: bool: {} } } + auth: { + description: "Azure service principal authentication." + required: false + type: object: options: { + azure_client_id: { + description: """ + The [Azure Client ID][azure_client_id]. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] + } + azure_client_secret: { + description: """ + The [Azure Client Secret][azure_client_secret]. + + [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] + } + azure_credential_kind: { + description: "The kind of Azure credential to use." + required: true + type: string: enum: { + azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" + managed_identity: "Use Managed Identity credentials" + managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" + workload_identity: "Use Workload Identity credentials" + } + } + azure_tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] + } + client_assertion_client_id: { + description: "The target Client ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_assertion_tenant_id: { + description: "The target Tenant ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } + user_assigned_managed_identity_id: { + description: "The User Assigned Managed Identity to use." + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } + } + } batch: { description: "Event batching behavior." required: false @@ -74,6 +201,16 @@ generated: components: sinks: azure_blob: configuration: { required: false type: bool: {} } + blob_endpoint: { + description: """ + The Azure Blob Storage endpoint. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["https://mylogstorage.blob.core.windows.net/"] + } blob_prefix: { description: """ A prefix to apply to all blob keys. @@ -165,8 +302,9 @@ generated: components: sinks: azure_blob: configuration: { | Allowed resource types | Container & Object | | Allowed permissions | Read & Create | """ - required: true - type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken"] + required: false + type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] + warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep connection strings secure and not expose them in logs, error messages, or version control systems."] } container_name: { description: "The Azure Blob Storage Account container name." @@ -862,4 +1000,17 @@ generated: components: sinks: azure_blob: configuration: { } } } + tls: { + description: "TLS configuration." + required: false + type: object: options: ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in PEM (X.509) format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + } } diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index ec03373e5354e..2aaf9b628e3c6 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -28,8 +28,8 @@ generated: components: sinks: azure_logs_ingestion: configuration: { } } auth: { - description: "Configuration of the authentication strategy for interacting with Azure services." - required: false + description: "Azure service principal authentication." + required: true type: object: options: { azure_client_id: { description: """ @@ -37,11 +37,9 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } azure_client_secret: { description: """ @@ -49,17 +47,17 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00-00~000000-0000000~0000000000000000000"] - } + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } azure_credential_kind: { description: "The kind of Azure credential to use." required: true type: string: enum: { azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -71,11 +69,21 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] } client_assertion_client_id: { description: "The target Client ID to use." @@ -89,12 +97,51 @@ generated: components: sinks: azure_logs_ingestion: configuration: { required: true type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } user_assigned_managed_identity_id: { - description: "The User Assigned Managed Identity (Client ID) to use." + description: "The User Assigned Managed Identity to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" required: false type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } } } batch: { From 985f669ffae07c1104ae9cec1b097e74130e0341 Mon Sep 17 00:00:00 2001 From: fcfangcc Date: Thu, 9 Apr 2026 01:50:38 +0800 Subject: [PATCH 065/364] fix(file source): high CPU usage after async file server migration (#25064) * fixed reached_eof * add changelog * update changlog --------- Co-authored-by: Pavlos Rontidis --- benches/files.rs | 113 +++++++++++++++++- ...fixed_high_cpu_usage_in_file_source.fix.md | 3 + lib/file-source-common/src/buffer.rs | 9 +- lib/file-source/src/file_watcher/mod.rs | 30 ++++- lib/file-source/src/file_watcher/tests/mod.rs | 68 ++++++++++- 5 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md diff --git a/benches/files.rs b/benches/files.rs index 703c4a984c8cd..6c6c00f367a66 100644 --- a/benches/files.rs +++ b/benches/files.rs @@ -1,9 +1,9 @@ use std::{convert::TryInto, path::PathBuf, time::Duration}; use bytes::Bytes; -use criterion::{BatchSize, Criterion, SamplingMode, Throughput, criterion_group}; +use criterion::{BatchSize, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group}; use futures::{SinkExt, StreamExt, stream}; -use tempfile::tempdir; +use tempfile::{TempDir, tempdir}; use tokio::fs::OpenOptions; use tokio_util::codec::{BytesCodec, FramedWrite}; use vector::{ @@ -12,6 +12,70 @@ use vector::{ }; use vector_lib::codecs::{TextSerializerConfig, encoding::FramingConfig}; +fn build_file_benchmark_environment( + idle_files: usize, +) -> ( + tokio::runtime::Runtime, + vector::topology::RunningTopology, + tokio::fs::File, + TempDir, +) { + let temp = tempdir().unwrap(); + let directory = temp.path().to_path_buf(); + + let mut data_dir = directory.clone(); + data_dir.push("data"); + std::fs::create_dir(&data_dir).unwrap(); + + let active_file = directory.join("active.log"); + std::fs::write(&active_file, []).unwrap(); + + for index in 0..idle_files { + let idle_file = directory.join(format!("idle-{index}.log")); + std::fs::write(idle_file, []).unwrap(); + } + + let output = directory.join("output.txt"); + + let mut source = sources::file::FileConfig::default(); + source.include = vec![directory.join("*.log")]; + source.data_dir = Some(data_dir); + + let mut config = config::Config::builder(); + config.add_source("in", source); + config.add_sink( + "out", + &["in"], + sinks::file::FileSinkConfig { + path: output.try_into().unwrap(), + idle_timeout: Duration::from_secs(30), + encoding: (None::, TextSerializerConfig::default()).into(), + compression: sinks::file::Compression::None, + acknowledgements: Default::default(), + timezone: Default::default(), + internal_metrics: Default::default(), + truncate: Default::default(), + }, + ); + + let rt = runtime(); + let (topology, input) = rt.block_on(async move { + let (topology, _crash) = start_topology(config.build().unwrap(), false).await; + + let mut options = OpenOptions::new(); + options.create(true).append(true).write(true); + + let input = options.open(active_file).await.unwrap(); + + // Give the source enough time to discover files and drive the idle watchers to EOF. + tokio::time::sleep(Duration::from_millis(250)).await; + + (topology, input) + }); + + (rt, topology, input, temp) +} + fn benchmark_files_no_partitions(c: &mut Criterion) { let num_lines: usize = 10_000; let line_size: usize = 100; @@ -99,10 +163,53 @@ fn benchmark_files_no_partitions(c: &mut Criterion) { group.finish(); } +fn benchmark_files_with_idle_watchers(c: &mut Criterion) { + let num_lines: usize = 10_000; + let line_size: usize = 100; + + let mut group = c.benchmark_group("files/idle_watchers"); + group.throughput(Throughput::Bytes((num_lines * line_size) as u64)); + group.sampling_mode(SamplingMode::Flat); + group.sample_size(10); + + for idle_files in [0usize, 128, 512] { + group.bench_with_input( + BenchmarkId::from_parameter(idle_files), + &idle_files, + |b, idle_files| { + b.iter_batched( + || build_file_benchmark_environment(*idle_files), + |(rt, topology, input, _temp)| { + rt.block_on(async move { + let mut sink = FramedWrite::new(input, BytesCodec::new()); + let raw_lines = + random_lines(line_size).take(num_lines).map(|mut line| { + line.push('\n'); + Bytes::from(line) + }); + let mut lines = stream::iter(raw_lines); + while let Some(line) = lines.next().await { + sink.send(line).await.unwrap(); + } + + // Keep the topology alive briefly so the benchmark includes watcher polling work. + tokio::time::sleep(Duration::from_millis(250)).await; + topology.stop().await; + }); + }, + BatchSize::LargeInput, + ) + }, + ); + } + + group.finish(); +} + criterion_group!( name = benches; // encapsulates inherent CI noise we saw in // https://github.com/vectordotdev/vector/issues/5394 config = Criterion::default().noise_threshold(0.05); - targets = benchmark_files_no_partitions + targets = benchmark_files_no_partitions, benchmark_files_with_idle_watchers ); diff --git a/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md b/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md new file mode 100644 index 0000000000000..8d64593e51b5c --- /dev/null +++ b/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md @@ -0,0 +1,3 @@ +Fixed an issue in the `file`/`kubernetes_logs` source that could cause unexpectedly high CPU usage after the async file server migration. + +authors: fcfangcc diff --git a/lib/file-source-common/src/buffer.rs b/lib/file-source-common/src/buffer.rs index 097331fbf2be3..099d896f4e1e6 100644 --- a/lib/file-source-common/src/buffer.rs +++ b/lib/file-source-common/src/buffer.rs @@ -1,5 +1,5 @@ use crate::FilePosition; -use std::{cmp::min, io, pin::Pin}; +use std::{cmp::min, io}; use bstr::Finder; use bytes::BytesMut; @@ -34,7 +34,7 @@ pub struct ReadResult { /// GiB/s range for buffers of length 1KiB. For buffers any smaller than this /// the overhead of setup dominates our benchmarks. pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( - reader: Pin>, + reader: &'a mut R, position: &'a mut FilePosition, delim: &'a [u8], buf: &'a mut BytesMut, @@ -45,7 +45,6 @@ pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( let delim_finder = Finder::new(delim); let delim_len = delim.len(); let mut discarded_for_size_and_truncated = Vec::new(); - let mut reader = Box::new(reader); // Used to track partial delimiter matches across buffer boundaries. // Data is read in chunks from the reader (see `fill_buf` below). @@ -294,7 +293,7 @@ mod test { let mut reader = BufReader::new(Cursor::new(&chunk)); match read_until_with_max_size( - Box::pin(&mut reader), + &mut reader, &mut position, &delimiter, &mut buffer, @@ -425,7 +424,7 @@ mod test { for (i, expected_line) in expected_lines.iter().enumerate() { let mut buffer = BytesMut::new(); let result = read_until_with_max_size( - Box::pin(&mut reader), + &mut reader, &mut position, delimiter, &mut buffer, diff --git a/lib/file-source/src/file_watcher/mod.rs b/lib/file-source/src/file_watcher/mod.rs index b750a81433886..6e103cf13f39b 100644 --- a/lib/file-source/src/file_watcher/mod.rs +++ b/lib/file-source/src/file_watcher/mod.rs @@ -19,6 +19,9 @@ use file_source_common::{ buffer::{ReadResult, read_until_with_max_size}, }; +const EOF_READ_BACKOFF_MIN: Duration = Duration::from_millis(1); +const EOF_READ_BACKOFF_MAX: Duration = Duration::from_millis(250); + #[cfg(test)] mod tests; @@ -57,6 +60,7 @@ pub struct FileWatcher { reached_eof: bool, last_read_attempt: Instant, last_read_success: Instant, + read_retry_delay: Duration, last_seen: Instant, max_line_bytes: usize, line_delimiter: Bytes, @@ -166,6 +170,7 @@ impl FileWatcher { reached_eof: false, last_read_attempt: ts, last_read_success: ts, + read_retry_delay: EOF_READ_BACKOFF_MIN, last_seen: ts, max_line_bytes, line_delimiter, @@ -196,6 +201,8 @@ impl FileWatcher { self.devno = file_info.portable_dev(); self.inode = file_info.portable_ino(); } + self.reached_eof = false; + self.read_retry_delay = EOF_READ_BACKOFF_MIN; self.path = path; Ok(()) } @@ -235,7 +242,7 @@ impl FileWatcher { let file_position = &mut self.file_position; let initial_position = *file_position; match read_until_with_max_size( - Box::pin(reader), + reader.as_mut(), file_position, self.line_delimiter.as_ref(), &mut self.buf, @@ -283,7 +290,7 @@ impl FileWatcher { }) } } else { - self.reached_eof = true; + self.track_read_eof(); Ok(RawLineResult { raw_line: None, discarded_for_size_and_truncated, @@ -306,9 +313,24 @@ impl FileWatcher { #[inline] fn track_read_success(&mut self) { + self.reached_eof = false; + self.read_retry_delay = EOF_READ_BACKOFF_MIN; self.last_read_success = Instant::now(); } + #[inline] + fn track_read_eof(&mut self) { + self.read_retry_delay = if self.reached_eof { + std::cmp::min( + self.read_retry_delay.saturating_mul(2), + EOF_READ_BACKOFF_MAX, + ) + } else { + EOF_READ_BACKOFF_MIN + }; + self.reached_eof = true; + } + #[inline] pub fn last_read_success(&self) -> Instant { self.last_read_success @@ -316,6 +338,10 @@ impl FileWatcher { #[inline] pub fn should_read(&self) -> bool { + if self.reached_eof && self.last_read_attempt.elapsed() < self.read_retry_delay { + return false; + } + self.last_read_success.elapsed() < Duration::from_secs(10) || self.last_read_attempt.elapsed() > Duration::from_secs(10) } diff --git a/lib/file-source/src/file_watcher/tests/mod.rs b/lib/file-source/src/file_watcher/tests/mod.rs index 7a9cc92909db0..5384a43802d24 100644 --- a/lib/file-source/src/file_watcher/tests/mod.rs +++ b/lib/file-source/src/file_watcher/tests/mod.rs @@ -1,9 +1,13 @@ mod experiment; mod experiment_no_truncations; -use std::str; +use std::{path::PathBuf, str, thread}; +use bytes::{Bytes, BytesMut}; use quickcheck::{Arbitrary, Gen}; +use tokio::time::Instant; + +use super::{EOF_READ_BACKOFF_MAX, EOF_READ_BACKOFF_MIN, FileWatcher, null_reader}; // Welcome. // @@ -171,6 +175,68 @@ impl Arbitrary for FileWatcherAction { } } +fn watcher_for_timing() -> FileWatcher { + let now = Instant::now(); + + FileWatcher { + path: PathBuf::new(), + findable: true, + reader: Box::new(null_reader()), + file_position: 0, + devno: 0, + inode: 0, + is_dead: false, + reached_eof: false, + last_read_attempt: now, + last_read_success: now, + read_retry_delay: EOF_READ_BACKOFF_MIN, + last_seen: now, + max_line_bytes: 1024, + line_delimiter: Bytes::from_static(b"\n"), + buf: BytesMut::new(), + } +} + +#[test] +fn backs_off_after_eof() { + let mut watcher = watcher_for_timing(); + + watcher.track_read_attempt(); + watcher.track_read_eof(); + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MIN); + assert!(!watcher.should_read()); + + thread::sleep(EOF_READ_BACKOFF_MIN); + + assert!(watcher.should_read()); + + watcher.track_read_attempt(); + watcher.track_read_eof(); + + assert_eq!( + watcher.read_retry_delay, + EOF_READ_BACKOFF_MIN.saturating_mul(2) + ); +} + +#[test] +fn caps_and_resets_eof_backoff() { + let mut watcher = watcher_for_timing(); + + for _ in 0..16 { + watcher.track_read_attempt(); + watcher.track_read_eof(); + } + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MAX); + + watcher.track_read_success(); + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MIN); + assert!(!watcher.reached_eof()); +} + #[inline] pub fn delay(attempts: u32) { let delay = match attempts { From 9c4abf31a91d888d0b4ca204643b0e6e1a559466 Mon Sep 17 00:00:00 2001 From: tronboto <142882846+tronboto@users.noreply.github.com> Date: Wed, 8 Apr 2026 21:26:19 +0100 Subject: [PATCH 066/364] fix(shutdown): emit VectorStopped after topology drains (#25083) * move VectorStopped event * add changelog * Add VectorStopping event * Update changelog.d/24532_vector_stopped_log_ordering.fix.md * Apply suggestion from @pront --------- Co-authored-by: Pavlos Rontidis --- .../24532_vector_stopped_log_ordering.fix.md | 5 ++++ src/app.rs | 25 +++++++++++-------- src/internal_events/process.rs | 12 +++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 changelog.d/24532_vector_stopped_log_ordering.fix.md diff --git a/changelog.d/24532_vector_stopped_log_ordering.fix.md b/changelog.d/24532_vector_stopped_log_ordering.fix.md new file mode 100644 index 0000000000000..45f1a82eea06f --- /dev/null +++ b/changelog.d/24532_vector_stopped_log_ordering.fix.md @@ -0,0 +1,5 @@ +Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. + +A new `VectorStoppping` event was added in the place of the `VectorStopped` event. + +authors: tronboto diff --git a/src/app.rs b/src/app.rs index 6f30f85de1d20..904d42be11968 100644 --- a/src/app.rs +++ b/src/app.rs @@ -28,7 +28,9 @@ use crate::{ config::{self, ComponentConfig, ComponentType, Config, ConfigPath}, extra_context::ExtraContext, heartbeat, - internal_events::{VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped}, + internal_events::{ + VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped, VectorStopping, + }, signal::{SignalHandler, SignalPair, SignalRx, SignalTo}, topology::{ ReloadOutcome, RunningTopology, SharedTopologyController, ShutdownErrorReceiver, @@ -493,16 +495,19 @@ impl FinishedApplication { } async fn stop(topology_controller: TopologyController, mut signal_rx: SignalRx) -> ExitStatus { - emit!(VectorStopped); + emit!(VectorStopping); tokio::select! { - _ = topology_controller.stop() => ExitStatus::from_raw({ - #[cfg(windows)] - { - exitcode::OK as u32 - } - #[cfg(unix)] - exitcode::OK - }), // Graceful shutdown finished + _ = topology_controller.stop() => { + emit!(VectorStopped); + ExitStatus::from_raw({ + #[cfg(windows)] + { + exitcode::OK as u32 + } + #[cfg(unix)] + exitcode::OK + }) + }, // Graceful shutdown finished _ = signal_rx.recv() => Self::quit(), } } diff --git a/src/internal_events/process.rs b/src/internal_events/process.rs index cc3609737aea9..987724c2da18d 100644 --- a/src/internal_events/process.rs +++ b/src/internal_events/process.rs @@ -38,6 +38,18 @@ impl InternalEvent for VectorReloaded<'_> { } } +#[derive(Debug, NamedInternalEvent)] +pub struct VectorStopping; + +impl InternalEvent for VectorStopping { + fn emit(self) { + info!( + target: "vector", + message = "Vector is stopping.", + ); + } +} + #[derive(Debug, NamedInternalEvent)] pub struct VectorStopped; From 26eef13087e1a463efa4bdcc9d3919b13936e06c Mon Sep 17 00:00:00 2001 From: Lisa Vu Date: Wed, 8 Apr 2026 16:26:27 -0400 Subject: [PATCH 067/364] fix(datadog_agent source): preserve `device` tag from v2 series resources (#25146) * fix(datadog_agent source): preserve `device` tag from v2 series resources The v2 series protobuf format encodes `device` in the `resources` array, but the decoder only special-cased `host`, causing `device` to be incorrectly remapped to `resource.device`. This broke tag continuity for disk, SNMP, and other integrations routing through OPW. * fix(changelog): use GitHub username for authors field The spell checker pattern for changelog authors only allows alphanumeric characters and hyphens. Changed from `lisa.vu` to `lisaqvu` (GitHub username) to match the expected format. --- .../datadog_agent_source_device_tag_v2.fix.md | 5 ++ src/sources/datadog_agent/metrics.rs | 4 ++ src/sources/datadog_agent/tests.rs | 60 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 changelog.d/datadog_agent_source_device_tag_v2.fix.md diff --git a/changelog.d/datadog_agent_source_device_tag_v2.fix.md b/changelog.d/datadog_agent_source_device_tag_v2.fix.md new file mode 100644 index 0000000000000..669d594f8c89f --- /dev/null +++ b/changelog.d/datadog_agent_source_device_tag_v2.fix.md @@ -0,0 +1,5 @@ +`datadog_agent` source: Preserve `device` as a plain tag when decoding v2 series metrics, +instead of incorrectly prefixing it as `resource.device`. This matches the v1 series behavior +and fixes tag remapping for disk, SNMP, and other integrations that use the `device` resource type. + +authors: lisaqvu diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index d931be05fcab7..6f377e826a8b8 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -289,6 +289,10 @@ pub(crate) fn decode_ddseries_v2( log_schema() .host_key() .and_then(|key| tags.replace(key.to_string(), r.name)); + } else if r.r#type.eq("device") { + // The `device` resource type is used by Agent checks (disk, SNMP/NDM, etc.) + // and must be preserved as a plain `device` tag to match the v1 series behavior. + tags.replace("device".into(), r.name); } else { // But to avoid losing information if this situation changes, any other resource type/name will be saved in the tags map tags.replace(format!("resource.{}", r.r#type), r.name); diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 065b6ec5f78f5..b2dbf9691db00 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2648,6 +2648,66 @@ async fn series_v2_split_metric_namespace_false() { .await; } +#[tokio::test] +async fn series_v2_device_resource_preserved_as_tag() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (rx, _, _, addr, _guard) = + source(EventStatus::Delivered, true, true, false, false).await; + + let series = vec![ddmetric_proto::metric_payload::MetricSeries { + resources: vec![ + ddmetric_proto::metric_payload::Resource { + r#type: "host".to_string(), + name: "test_host".to_string(), + }, + ddmetric_proto::metric_payload::Resource { + r#type: "device".to_string(), + name: "sda".to_string(), + }, + ], + metric: "system.disk.free".to_string(), + tags: vec!["env:prod".to_string()], + points: vec![ddmetric_proto::metric_payload::MetricPoint { + value: 100.0, + timestamp: 1542182950, + }], + r#type: ddmetric_proto::metric_payload::MetricType::Gauge as i32, + unit: "".to_string(), + source_type_name: "".to_string(), + interval: 0, + metadata: None, + }]; + + let series_payload = ddmetric_proto::MetricPayload { series }; + let mut buf = Vec::new(); + series_payload.encode(&mut buf).unwrap(); + let body = unsafe { String::from_utf8_unchecked(buf) }; + + let events = send_and_collect( + addr, + body, + dd_api_key_headers(), + DD_API_SERIES_V2_PATH, + rx, + 1, + ) + .await; + + let metric = events[0].as_metric(); + let tags = metric.tags().unwrap(); + + // The `device` resource type must be preserved as a plain `device` tag, + // NOT as `resource.device`. This matches v1 series behavior. + assert_eq!(tags.get("device"), Some("sda")); + assert!( + tags.get("resource.device").is_none(), + "device should not be prefixed with 'resource.'" + ); + assert_eq!(tags.get("env"), Some("prod")); + }) + .await; +} + async fn test_sketches_split_metric_namespace_impl( split: bool, expected_name: &str, From b57d8b0ef0ec12a1645b556a6b1bf080dc66e50b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 20:53:19 +0000 Subject: [PATCH 068/364] chore(deps): bump basic-ftp from 5.2.0 to 5.2.1 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory (#25147) chore(deps): bump basic-ftp Bumps the npm_and_yarn group with 1 update in the /scripts/environment/npm-tools directory: [basic-ftp](https://github.com/patrickjuchli/basic-ftp). Updates `basic-ftp` from 5.2.0 to 5.2.1 - [Release notes](https://github.com/patrickjuchli/basic-ftp/releases) - [Changelog](https://github.com/patrickjuchli/basic-ftp/blob/master/CHANGELOG.md) - [Commits](https://github.com/patrickjuchli/basic-ftp/compare/v5.2.0...v5.2.1) --- updated-dependencies: - dependency-name: basic-ftp dependency-version: 5.2.1 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- scripts/environment/npm-tools/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index 1047d5444f89a..708206c20ade0 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -596,9 +596,9 @@ "license": "MIT" }, "node_modules/basic-ftp": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", - "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.1.tgz", + "integrity": "sha512-0yaL8JdxTknKDILitVpfYfV2Ob6yb3udX/hK97M7I3jOeznBNxQPtVvTUtnhUkyHlxFWyr5Lvknmgzoc7jf+1Q==", "license": "MIT", "engines": { "node": ">=10.0.0" From 51f6fce6d850dc4c6833d910013a189119068734 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 9 Apr 2026 13:35:17 +0000 Subject: [PATCH 069/364] chore(shutdown): fix typo in 24532 changelog fragment (#25155) --- changelog.d/24532_vector_stopped_log_ordering.fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/24532_vector_stopped_log_ordering.fix.md b/changelog.d/24532_vector_stopped_log_ordering.fix.md index 45f1a82eea06f..c5e395c37ab6f 100644 --- a/changelog.d/24532_vector_stopped_log_ordering.fix.md +++ b/changelog.d/24532_vector_stopped_log_ordering.fix.md @@ -1,5 +1,5 @@ Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. -A new `VectorStoppping` event was added in the place of the `VectorStopped` event. +A new `VectorStopping` event was added in the place of the `VectorStopped` event. authors: tronboto From a7f6487c94b98766bdff802aac0cd1372b0311f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:07:40 +0000 Subject: [PATCH 070/364] chore(deps): bump basic-ftp from 5.2.1 to 5.2.2 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory (#25170) chore(deps): bump basic-ftp Bumps the npm_and_yarn group with 1 update in the /scripts/environment/npm-tools directory: [basic-ftp](https://github.com/patrickjuchli/basic-ftp). Updates `basic-ftp` from 5.2.1 to 5.2.2 - [Release notes](https://github.com/patrickjuchli/basic-ftp/releases) - [Changelog](https://github.com/patrickjuchli/basic-ftp/blob/master/CHANGELOG.md) - [Commits](https://github.com/patrickjuchli/basic-ftp/compare/v5.2.1...v5.2.2) --- updated-dependencies: - dependency-name: basic-ftp dependency-version: 5.2.2 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- scripts/environment/npm-tools/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index 708206c20ade0..9e25b17830d18 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -596,9 +596,9 @@ "license": "MIT" }, "node_modules/basic-ftp": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.1.tgz", - "integrity": "sha512-0yaL8JdxTknKDILitVpfYfV2Ob6yb3udX/hK97M7I3jOeznBNxQPtVvTUtnhUkyHlxFWyr5Lvknmgzoc7jf+1Q==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.2.tgz", + "integrity": "sha512-1tDrzKsdCg70WGvbFss/ulVAxupNauGnOlgpyjKzeQxzyllBLS0CGLV7tjIXTK3ZQA9/FBEm9qyFFN1bciA6pw==", "license": "MIT", "engines": { "node": ">=10.0.0" From 5ab0ba0d27392c70995a3d5e182a8307e65c641f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:37:29 -0400 Subject: [PATCH 071/364] chore(website deps): bump axios from 1.13.5 to 1.15.0 in /website (#25172) Bumps [axios](https://github.com/axios/axios) from 1.13.5 to 1.15.0. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.13.5...v1.15.0) --- updated-dependencies: - dependency-name: axios dependency-version: 1.15.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 055e51daa030a..31254815a65f1 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -1641,13 +1641,13 @@ autoprefixer@^10.2.5: postcss-value-parser "^4.2.0" axios@^1.6.0, axios@^1.8.4: - version "1.13.5" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.13.5.tgz#5e464688fa127e11a660a2c49441c009f6567a43" - integrity sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q== + version "1.15.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.15.0.tgz#0fcee91ef03d386514474904b27863b2c683bf4f" + integrity sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q== dependencies: follow-redirects "^1.15.11" form-data "^4.0.5" - proxy-from-env "^1.1.0" + proxy-from-env "^2.1.0" babel-plugin-macros@^3.1.0: version "3.1.0" @@ -3190,10 +3190,10 @@ prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.13.1" -proxy-from-env@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz" - integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== +proxy-from-env@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" + integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== purgecss@^4.0.3, purgecss@^4.1.3: version "4.1.3" From 480ca58f52a019454a36e9c9b754791224fcee57 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 13 Apr 2026 11:22:13 -0400 Subject: [PATCH 072/364] chore(vdev): extract vector-vrl-doc-builder into unpublished crate (#25034) * chore(vdev): extract vector-vrl-doc-builder into unpublished crate * fix(ci): address issues caught by Codex code review Allow clippy::print_stdout on the doc-builder binary (the workspace-wide lint denial would otherwise fail clippy) and explicitly request Rust in the VRL docs CI workflow so it does not rely on cache state. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(ci): update VRL doc-builder workflow filters, caching, and CLI modes - Replace stale vdev/** path filter with Makefile, drop vdev: true - Add install-vrl-doc-builder action with binary caching (mirrors vdev pattern) - Use clap conflicts_with/requires for mutually exclusive CLI flags - Makefile auto-detects installed binary, falls back to cargo run Co-Authored-By: Claude Opus 4.6 (1M context) * fix(ci): add protoc to VRL doc-builder workflow vector-core's build.rs requires protoc for proto compilation. Previously protoc was installed implicitly via VDEV_NEEDS_COMPILE; now that vdev is disabled we need to request it explicitly. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(ci): drop redundant false overrides from setup action call mold and cargo-cache default to true which is fine; no need to explicitly disable them. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.6 (1M context) --- .../install-vrl-doc-builder/action.yml | 25 ++++++++++ .../workflows/check_generated_vrl_docs.yml | 10 ++-- Cargo.lock | 13 ++++- Cargo.toml | 1 + Makefile | 9 +++- lib/vector-vrl/doc-builder/Cargo.toml | 14 ++++++ lib/vector-vrl/doc-builder/src/main.rs | 49 +++++++++++++++++++ vdev/Cargo.toml | 2 - vdev/src/commands/build/mod.rs | 2 - vdev/src/commands/build/vector_vrl_docs.rs | 47 ------------------ 10 files changed, 114 insertions(+), 58 deletions(-) create mode 100644 .github/actions/install-vrl-doc-builder/action.yml create mode 100644 lib/vector-vrl/doc-builder/Cargo.toml create mode 100644 lib/vector-vrl/doc-builder/src/main.rs delete mode 100644 vdev/src/commands/build/vector_vrl_docs.rs diff --git a/.github/actions/install-vrl-doc-builder/action.yml b/.github/actions/install-vrl-doc-builder/action.yml new file mode 100644 index 0000000000000..3748e02e8b295 --- /dev/null +++ b/.github/actions/install-vrl-doc-builder/action.yml @@ -0,0 +1,25 @@ +name: "Install vector-vrl-doc-builder" +description: "Install vector-vrl-doc-builder CLI tool with caching based on source code changes" + +branding: + icon: tool + color: purple + +runs: + using: "composite" + steps: + - name: Cache vrl-doc-builder binary + id: cache-vrl-doc-builder + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.cargo/bin/vector-vrl-doc-builder + key: ${{ runner.os }}-vrl-doc-builder-${{ hashFiles('lib/vector-vrl/**', 'Cargo.toml', 'Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-vrl-doc-builder- + + - name: Build and install vector-vrl-doc-builder + if: steps.cache-vrl-doc-builder.outputs.cache-hit != 'true' + shell: bash + run: | + echo "Building vector-vrl-doc-builder from source (cache miss)" + cargo install --path lib/vector-vrl/doc-builder --locked --force diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index e183e12578f6e..c2a9852474461 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -34,7 +34,7 @@ jobs: filters: | docs: - "lib/vector-vrl/**" - - "vdev/**" + - "Makefile" - "Cargo.lock" - ".github/workflows/check_generated_vrl_docs.yml" @@ -51,9 +51,11 @@ jobs: - uses: ./.github/actions/setup with: - vdev: true - mold: false - cargo-cache: false + rust: true + protoc: true + vdev: false + + - uses: ./.github/actions/install-vrl-doc-builder - name: Regenerate VRL docs run: make generate-vector-vrl-docs diff --git a/Cargo.lock b/Cargo.lock index 45f4a998276fc..054d3afc66007 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12226,8 +12226,6 @@ dependencies = [ "tempfile", "toml", "toml_edit 0.23.9", - "vector-vrl-functions", - "vrl", ] [[package]] @@ -12788,6 +12786,17 @@ dependencies = [ "vrl", ] +[[package]] +name = "vector-vrl-doc-builder" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde_json", + "vector-vrl-functions", + "vrl", +] + [[package]] name = "vector-vrl-functions" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 25ca1f9e0dec9..9389c9420d248 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,6 +127,7 @@ members = [ "lib/vector-vrl/category", "lib/vector-vrl/cli", "lib/vector-vrl/dnstap-parser", + "lib/vector-vrl/doc-builder", "lib/vector-vrl/enrichment", "lib/vector-vrl/functions", "lib/vector-vrl/metrics", diff --git a/Makefile b/Makefile index a923806254d9f..34da1302487d9 100644 --- a/Makefile +++ b/Makefile @@ -734,9 +734,16 @@ generate-component-docs: ## Generate per-component Cue docs from the configurati $(if $(findstring true,$(CI)),>/dev/null,) ./scripts/cue.sh fmt +VRL_DOC_BUILDER := $(shell command -v vector-vrl-doc-builder 2>/dev/null) +ifndef VRL_DOC_BUILDER +VRL_DOC_BUILDER_CMD = cargo run -p vector-vrl-doc-builder -- +else +VRL_DOC_BUILDER_CMD = vector-vrl-doc-builder +endif + .PHONY: generate-vector-vrl-docs generate-vector-vrl-docs: ## Generate VRL function documentation from Rust source. - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build vector-vrl-docs --output docs/generated/ \ + ${MAYBE_ENVIRONMENT_EXEC} $(VRL_DOC_BUILDER_CMD) --output docs/generated/ \ $(if $(findstring true,$(CI)),>/dev/null,) .PHONY: generate-vrl-docs diff --git a/lib/vector-vrl/doc-builder/Cargo.toml b/lib/vector-vrl/doc-builder/Cargo.toml new file mode 100644 index 0000000000000..75a1d262f4708 --- /dev/null +++ b/lib/vector-vrl/doc-builder/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vector-vrl-doc-builder" +version = "0.1.0" +authors = ["Vector Contributors "] +edition = "2024" +publish = false +license = "MPL-2.0" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +vector-vrl-functions = { path = "../functions", features = ["dnstap", "vrl-metrics"] } +vrl = { workspace = true, features = ["docs"] } diff --git a/lib/vector-vrl/doc-builder/src/main.rs b/lib/vector-vrl/doc-builder/src/main.rs new file mode 100644 index 0000000000000..92c38dfea307d --- /dev/null +++ b/lib/vector-vrl/doc-builder/src/main.rs @@ -0,0 +1,49 @@ +use anyhow::Result; +use std::path::PathBuf; +use vrl::docs::{build_functions_doc, document_functions_to_dir}; + +/// Generate Vector-specific VRL function documentation as JSON files. +/// +/// Two modes of operation: +/// --output Write one JSON file per function into DIR (uses --extension). +/// (no --output) Print all functions to stdout as a JSON array (uses --minify). +#[derive(clap::Parser, Debug)] +#[command()] +struct Cli { + /// Output directory to create JSON files. When omitted, output is written to stdout as a JSON + /// array. + #[arg(short, long, conflicts_with = "minify")] + output: Option, + + /// Whether to minify the JSON output (stdout mode only) + #[arg(short, long, default_value_t = false, conflicts_with = "output")] + minify: bool, + + /// File extension for generated files (directory mode only) + #[arg(short, long, default_value = "json", requires = "output")] + extension: String, +} + +#[allow(clippy::print_stdout)] +fn main() -> Result<()> { + let cli = ::parse(); + let functions = vector_vrl_functions::all_without_vrl_stdlib(); + if let Some(output) = &cli.output { + document_functions_to_dir(&functions, output, &cli.extension)?; + } else { + let built = build_functions_doc(&functions); + if cli.minify { + println!( + "{}", + serde_json::to_string(&built).expect("FunctionDoc serialization should not fail") + ); + } else { + println!( + "{}", + serde_json::to_string_pretty(&built) + .expect("FunctionDoc serialization should not fail") + ); + } + } + Ok(()) +} diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 8fea6eb281d0a..7de3fc06d5a59 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -46,8 +46,6 @@ strum.workspace = true indoc.workspace = true git2 = { version = "0.20.4" } cfg-if.workspace = true -vector-vrl-functions = { path = "../lib/vector-vrl/functions", features = ["dnstap", "vrl-metrics"] } -vrl = { workspace = true, features = ["docs"] } [package.metadata.binstall] pkg-url = "{ repo }/releases/download/vdev-v{ version }/{ name }-{ target }-v{ version }.tgz" diff --git a/vdev/src/commands/build/mod.rs b/vdev/src/commands/build/mod.rs index 845ae928b84d9..f46d8a8ae210c 100644 --- a/vdev/src/commands/build/mod.rs +++ b/vdev/src/commands/build/mod.rs @@ -1,7 +1,6 @@ mod licenses; mod publish_metadata; mod vector; -mod vector_vrl_docs; mod vrl_docs; mod vrl_wasm; @@ -14,7 +13,6 @@ crate::cli_subcommands! { release_cue, vector, vrl_docs, - vector_vrl_docs, vrl_wasm, } diff --git a/vdev/src/commands/build/vector_vrl_docs.rs b/vdev/src/commands/build/vector_vrl_docs.rs deleted file mode 100644 index 7c25a23b139c3..0000000000000 --- a/vdev/src/commands/build/vector_vrl_docs.rs +++ /dev/null @@ -1,47 +0,0 @@ -use anyhow::Result; -use std::path::PathBuf; -use vrl::docs::{build_functions_doc, document_functions_to_dir}; - -/// Generate Vector-specific VRL function documentation as JSON files. -#[derive(clap::Args, Debug)] -#[command()] -pub struct Cli { - /// Output directory to create JSON files. If unspecified output is written to stdout as a JSON - /// array - #[arg(short, long)] - output: Option, - - /// Whether to pretty-print or minify - #[arg(short, long, default_value_t = false)] - minify: bool, - - /// File extension for generated files - #[arg(short, long, default_value = "json")] - extension: String, -} - -impl Cli { - pub fn exec(self) -> Result<()> { - let functions = vector_vrl_functions::all_without_vrl_stdlib(); - if let Some(output) = &self.output { - document_functions_to_dir(&functions, output, &self.extension)?; - } else { - let built = build_functions_doc(&functions); - #[allow(clippy::print_stdout)] - if self.minify { - println!( - "{}", - serde_json::to_string(&built) - .expect("FunctionDoc serialization should not fail") - ); - } else { - println!( - "{}", - serde_json::to_string_pretty(&built) - .expect("FunctionDoc serialization should not fail") - ); - } - } - Ok(()) - } -} From 3112033384d47b0eae9fa36a1297dab32cac1f88 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 13 Apr 2026 12:08:25 -0400 Subject: [PATCH 073/364] docs(website): convert TOML config examples to YAML (#25163) * docs(website): convert TOML config examples to YAML across website YAML is Vector's recommended and default configuration format. This converts all TOML-only config examples in website docs, guides, blog posts, and highlight files to YAML for consistency. Co-Authored-By: Claude Opus 4.6 (1M context) * docs(website): add YAML recommendation to multi-format references Update docs that mention TOML/YAML/JSON support to explicitly recommend YAML as the default configuration format. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: add YAML-default guidance to AGENTS.md Instructs AI tools to generate Vector config in YAML unless explicitly asked otherwise. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: remove version reference from AGENTS.md YAML guidance Co-Authored-By: Claude Opus 4.6 (1M context) * docs(website): fix issues from code review - Fix invalid YAML escape in template-syntax.md (use single quotes) - Fix `transform:` typo to `transforms:` in 0-25-0-upgrade-guide.md - Update remaining `vector.toml` prose references to `vector.yaml` - Remove anachronistic version reference from 2020 highlight Co-Authored-By: Claude Opus 4.6 (1M context) * docs(website): add YAML/TOML format picker tabs to template-syntax.md Add tabbed config format picker to all examples so users can toggle between YAML and TOML. Defaults to YAML. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- AGENTS.md | 4 + .../en/blog/adaptive-request-concurrency.md | 16 +- website/content/en/blog/graphql-api.md | 39 +- .../content/en/blog/kubernetes-integration.md | 20 +- website/content/en/blog/log-namespacing.md | 45 +- .../en/docs/administration/management.md | 2 +- .../en/docs/architecture/guarantees.md | 13 +- .../en/docs/architecture/pipeline-model.md | 2 +- .../en/docs/reference/configuration/_index.md | 5 +- .../configuration/template-syntax.md | 90 +++ .../reference/configuration/unit-tests.md | 528 ++++++++-------- .../setup/going-to-prod/high-availability.md | 37 +- .../advanced/custom-aggregations-with-lua.md | 245 ++++---- .../advanced/merge-multiline-logs-with-lua.md | 72 +-- .../advanced/parsing-csv-logs-with-lua.md | 193 +++--- .../en/guides/aws/cloudwatch-logs-firehose.md | 117 ++-- .../level-up/managing-complex-configs.md | 217 +++---- .../en/guides/level-up/managing-schemas.md | 232 +++---- .../en/guides/level-up/vector-tap-guide.md | 121 ++-- ...-11-25-unit-testing-vector-config-files.md | 57 +- .../en/highlights/2019-12-13-custom-dns.md | 4 +- .../en/highlights/2019-12-16-ec2-metadata.md | 26 +- .../2020-01-07-prometheus-source.md | 11 +- ...01-20-splunk-hec-specify-indexed-fields.md | 11 +- .../2020-02-05-merge-partial-docker-events.md | 9 +- .../2020-02-14-global-log-schema.md | 10 +- ...020-02-21-file-source-multiline-support.md | 20 +- .../2020-02-24-swimlanes-transform.md | 19 +- ...3-04-encoding-only-fields-except-fields.md | 16 +- .../highlights/2020-03-10-dedupe-transform.md | 16 +- ...0-03-11-tag-cardinality-limit-transform.md | 15 +- .../highlights/2020-03-31-filter-transform.md | 18 +- .../2020-04-01-more-condition-predicates.md | 10 +- ...27-add-support-for-loading-multiple-cas.md | 20 +- .../2020-07-10-add-reduce-transform.md | 79 +-- .../2020-09-18-adaptive-concurrency.md | 12 +- .../2020-10-27-metrics-integrations.md | 14 +- ...20-11-19-prometheus-remote-integrations.md | 39 +- .../2020-11-25-json-yaml-config-formats.md | 6 +- .../en/highlights/2020-12-23-graphql-api.md | 8 +- .../2020-12-23-internal-logs-source.md | 39 +- .../2021-01-10-kafka-sink-metrics.md | 17 +- .../2021-01-20-wildcard-identifiers.md | 34 +- .../2021-02-16-0-12-upgrade-guide.md | 12 +- .../2021-02-16-filter-remap-support.md | 34 +- .../en/highlights/2021-04-21-vector-tap.md | 29 +- .../en/highlights/2021-04-21-vrl-abort.md | 43 +- .../2021-07-14-0-15-upgrade-guide.md | 102 ++-- .../en/highlights/2021-07-14-vector-graph.md | 104 ++-- .../highlights/2021-07-16-remap-multiple.md | 65 +- .../en/highlights/2021-08-20-rate-limits.md | 7 +- .../2021-08-25-0-16-upgrade-guide.md | 32 +- .../en/highlights/2021-10-06-arc-default.md | 10 +- .../en/highlights/2021-10-06-source-codecs.md | 29 +- .../2021-11-12-event-throttle-transform.md | 13 +- .../2021-12-15-splunk-hec-improvements.md | 9 +- .../2021-12-28-0-19-0-upgrade-guide.md | 14 +- ...022-01-12-vector-unit-test-improvements.md | 44 +- .../2022-02-08-disk-buffer-v2-beta.md | 28 +- .../en/highlights/2022-03-15-vrl-vm-beta.md | 16 +- .../2022-03-22-0-21-0-upgrade-guide.md | 26 +- .../2022-03-31-native-event-codecs.md | 44 +- .../2022-05-03-0-22-0-upgrade-guide.md | 572 +++++++++--------- .../2022-07-07-0-23-0-upgrade-guide.md | 73 ++- .../2022-07-07-secrets-management.md | 25 +- .../en/highlights/2022-07-07-sink-codecs.md | 17 +- .../2022-10-04-0-25-0-upgrade-guide.md | 60 +- .../2024-07-29-0-40-0-upgrade-guide.md | 9 +- 68 files changed, 2096 insertions(+), 1829 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d25eca40144c..118ebb7461dbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,6 +147,10 @@ make cue-build **Note:** Website changes use Hugo, CUE, Tailwind CSS, and TypeScript. See [website/README.md](website/README.md) for details. +## Configuration Format + +Always generate Vector configuration examples in **YAML** unless the user explicitly asks for TOML or JSON. YAML is Vector's recommended and default configuration format. + ## Common Patterns ### Development Tools diff --git a/website/content/en/blog/adaptive-request-concurrency.md b/website/content/en/blog/adaptive-request-concurrency.md index b2a701433d638..623d93d3d7749 100644 --- a/website/content/en/blog/adaptive-request-concurrency.md +++ b/website/content/en/blog/adaptive-request-concurrency.md @@ -94,13 +94,15 @@ This decision tree is always active and Vector always "knows" what to do, even i Vector never stops quietly making the linear up vs. exponential down decision in the background, and it works out of the box with zero configuration beyond enabling the feature, which is currently on an opt-in basis in version 0.11. You can enable ARC in an HTTP sink by setting the [`request.concurrency`][request_concurrency] parameter to `adaptive`. Here's an example for a Clickhouse sink: -```toml -[sinks.clickhouse_internal] -type = "clickhouse" -inputs = ["log_stream_1", "log_stream_2"] -host = "http://clickhouse-prod:8123" -table = "prod-log-data" -request.concurrency = "adaptive" +```yaml +sinks: + clickhouse_internal: + type: "clickhouse" + inputs: ["log_stream_1", "log_stream_2"] + host: "http://clickhouse-prod:8123" + table: "prod-log-data" + request: + concurrency: "adaptive" ``` There's also room for fine-tuning if you find yourself needing additional knobs: diff --git a/website/content/en/blog/graphql-api.md b/website/content/en/blog/graphql-api.md index f623c4caa921d..6d459dda9de95 100644 --- a/website/content/en/blog/graphql-api.md +++ b/website/content/en/blog/graphql-api.md @@ -36,19 +36,22 @@ tuned! Start up Vector locally with the following config: -```toml -[api] -enabled = true - -[sources.demo] -type = "demo_logs" -format = "json" -interval = 1.0 - -[sinks.console] -type = "console" -inputs = ["demo"] -encoding.codec = "text" +```yaml +api: + enabled: true + +sources: + demo: + type: "demo_logs" + format: "json" + interval: 1.0 + +sinks: + console: + type: "console" + inputs: ["demo"] + encoding: + codec: "text" ``` You can then access @@ -190,12 +193,12 @@ query { The API is an opt-in feature that spawns an HTTP server to accept queries. -To enable, open your `vector.toml`, and add the following: +To enable, open your `vector.yaml`, and add the following: -```toml -[api] - enabled = true - address = "127.0.0.1:8686" # optional. Change IP/port if required +```yaml +api: + enabled: true + address: "127.0.0.1:8686" # optional. Change IP/port if required ``` After restarting Vector, you can access an API playground at diff --git a/website/content/en/blog/kubernetes-integration.md b/website/content/en/blog/kubernetes-integration.md index bf60365d98ed1..bf9d7fbbb1cfa 100644 --- a/website/content/en/blog/kubernetes-integration.md +++ b/website/content/en/blog/kubernetes-integration.md @@ -27,15 +27,17 @@ The crux of our Kubernetes integration for Vector is the new [`kubernetes_logs`] This simple Vector configuration, for example, would suffice to send logs from *all* of your Nodes/Pods to [Elasticsearch]: -```toml title="vector.toml" -[sources.k8s_all] -type = "kubernetes_logs" - -[sinks.es_out] -type = "elasticsearch" -inputs = ["k8s_all"] -host = "http://your-elasticsearch-cluster:9200" -index = "vector-k8s-%F" +```yaml title="vector.yaml" +sources: + k8s_all: + type: "kubernetes_logs" + +sinks: + es_out: + type: "elasticsearch" + inputs: ["k8s_all"] + host: "http://your-elasticsearch-cluster:9200" + index: "vector-k8s-%F" ``` Using this simple topology as a starting point, you can [transform][transforms] your log data in a wide variety of ways, ship it to [other sinks][sinks], and much more. diff --git a/website/content/en/blog/log-namespacing.md b/website/content/en/blog/log-namespacing.md index d867f21c9411d..2d7ce9b314e58 100644 --- a/website/content/en/blog/log-namespacing.md +++ b/website/content/en/blog/log-namespacing.md @@ -39,27 +39,30 @@ so you can try out Log Namespacing on individual sources. The following example enables the `log_namespace` feature globally, then disables it for a single source. -```toml -schema.log_namespace = true - -[sources.input_with_log_namespace] -type = "demo_logs" -format = "shuffle" -lines = ["input_with_log_namespace"] -interval = 1 - -[sources.input_without_log_namespace] -type = "demo_logs" -format = "shuffle" -lines = ["input_without_log_namespace"] -interval = 1 -log_namespace = false - -[sinks.console] -type = "console" -inputs = ["input_with_log_namespace", "input_without_log_namespace"] -encoding.codec = "json" - +```yaml +schema: + log_namespace: true + +sources: + input_with_log_namespace: + type: "demo_logs" + format: "shuffle" + lines: ["input_with_log_namespace"] + interval: 1 + + input_without_log_namespace: + type: "demo_logs" + format: "shuffle" + lines: ["input_without_log_namespace"] + interval: 1 + log_namespace: false + +sinks: + console: + type: "console" + inputs: ["input_with_log_namespace", "input_without_log_namespace"] + encoding: + codec: "json" ``` ## How It Works diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index dff4be7153bbf..7f91c26cdd191 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -176,7 +176,7 @@ docker restart -f $(docker ps -aqf "name=vector") {{< /tab >}} {{< /tabs >}} -The commands above involve configuring Vector using TOML, but you can also use JSON or YAML. You can also use one of +The commands above involve configuring Vector using YAML, but you can also use TOML or JSON. You can also use one of three image variants (the commands assume `alpine`): | Variant | Image basis | diff --git a/website/content/en/docs/architecture/guarantees.md b/website/content/en/docs/architecture/guarantees.md index 76028d4e42fe1..14a1c0c9a72d8 100644 --- a/website/content/en/docs/architecture/guarantees.md +++ b/website/content/en/docs/architecture/guarantees.md @@ -61,12 +61,13 @@ delivery?](#faq-at-least-once) FAQ below). In order to achieve at-least-once delivery between restarts, your sink must be configured to use disk-based buffers: -```toml title="vector.toml" -[sinks.my_sink_id] - [sinks.my_sink_id.buffer] - type = "disk" - when_full = "block" - max_size = 104900000 # 100MiB +```yaml title="vector.yaml" +sinks: + my_sink_id: + buffer: + type: "disk" + when_full: "block" + max_size: 104900000 # 100MiB ``` Refer to each [sink's][sinks] documentation for further guidance on its buffer options. diff --git a/website/content/en/docs/architecture/pipeline-model.md b/website/content/en/docs/architecture/pipeline-model.md index 49cc846f7ac96..ddb6d84bc3b8c 100644 --- a/website/content/en/docs/architecture/pipeline-model.md +++ b/website/content/en/docs/architecture/pipeline-model.md @@ -10,7 +10,7 @@ Vector's pipeline model is based on a [directed acyclic graph][dag] of [componen ## Defining pipelines -A Vector pipeline is defined through a YAML, TOML, or JSON [configuration] file. For maintainability, many Vector users use configuration and data templating languages like [Jsonnet] or [CUE]. +A Vector pipeline is defined through a YAML, TOML, or JSON [configuration] file. We recommend using YAML as the default format. For maintainability, many Vector users use configuration and data templating languages like [Jsonnet] or [CUE]. Configuration is checked at pipeline compile time (when Vector boots). This prevents simple mistakes and enforces DAG properties. diff --git a/website/content/en/docs/reference/configuration/_index.md b/website/content/en/docs/reference/configuration/_index.md index 9e19281d9024c..65782a59b6000 100644 --- a/website/content/en/docs/reference/configuration/_index.md +++ b/website/content/en/docs/reference/configuration/_index.md @@ -250,8 +250,9 @@ Refer to the [Environment Variables](docs/reference/environment_variables/) page ### Formats Vector supports [YAML], [TOML], and [JSON] to ensure that Vector fits into your -workflow. A side benefit of supporting YAML and JSON is that they enable you to use -data templating languages such as [ytt], [Jsonnet] and [Cue]. +workflow. We recommend using YAML as the default configuration format. A side +benefit of supporting YAML and JSON is that they enable you to use data templating +languages such as [ytt], [Jsonnet] and [Cue]. #### Location diff --git a/website/content/en/docs/reference/configuration/template-syntax.md b/website/content/en/docs/reference/configuration/template-syntax.md index 43822e137c555..96135c0183745 100644 --- a/website/content/en/docs/reference/configuration/template-syntax.md +++ b/website/content/en/docs/reference/configuration/template-syntax.md @@ -12,6 +12,20 @@ data. Any option that supports this syntax will be clearly documented as such in Let's partition data on AWS S3 by "application_id" and "date". We can accomplish this with the `key_prefix` option in the `aws_s3` sink: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +sinks: + backup: + type: "aws_s3" + bucket: "all_application_logs" + key_prefix: "application_id={{ application_id }}/date=%F/" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml [sinks.backup] type = "aws_s3" @@ -19,6 +33,9 @@ the `aws_s3` sink: key_prefix = "application_id={{ application_id }}/date=%F/" ``` +{{< /tab >}} +{{< /tabs >}} + Notice that Vector allows direct field references as well as "strftime" specifiers. If we were to run the following log event through Vector: @@ -45,18 +62,44 @@ enables dynamic partitioning, something fundamental to storing log data in files Individual [log event][log] fields can be accessed using `{{ ... }}` to wrap a VRL [path expression][path_expression]: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: "{{ .parent.child }}" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "{{ .parent.child }}" ``` +{{< /tab >}} +{{< /tabs >}} + ### Strftime specifiers In addition to directly accessing fields, Vector offers a shortcut for injecting [strftime specifiers][strftime]: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: "year=%Y/month=%m/day=%d/" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "year=%Y/month=%m/day=%d/" ``` +{{< /tab >}} +{{< /tabs >}} + {{< info >}} The value is derived from the [`timestamp` field](/docs/architecture/data-model/log/#timestamps) and the name of this field can be changed via the [global `timestamp_key` option](/docs/reference/configuration/schema/#log_schema.timestamp_key). @@ -67,16 +110,42 @@ and the name of this field can be changed via the [global `timestamp_key` option You can escape this syntax by prefixing the character with a `\`. For example, you can escape the event field syntax like this: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: '\{{ field_name }}' +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "\{{ field_name }}" ``` +{{< /tab >}} +{{< /tabs >}} + And [strftime] specified like so: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: "year=\\%Y/month=\\%m/day=\\%d/" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "year=\%Y/month=\%m/day=\%d/" ``` +{{< /tab >}} +{{< /tabs >}} + Each of the values above would be treated literally. ## How it works @@ -91,6 +160,24 @@ You can find additional examples for accessing fields in the Vector doesn't currently support fallback values, [issue 1692][1692] is open to add this functionality. In the interim, you can use the [`remap` transform][remap] to set a default value: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +transforms: + set_defaults: + type: "remap" + inputs: + - "my-source-id" + source: | + if !exists(.my_field) { + .my_field = "default" + } +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml [transforms.set_defaults] type = "remap" @@ -102,6 +189,9 @@ you can use the [`remap` transform][remap] to set a default value: ''' ``` +{{< /tab >}} +{{< /tabs >}} + ### Missing fields If a field is missing, an error is logged and Vector drops the event. The `component_errors_total` internal diff --git a/website/content/en/docs/reference/configuration/unit-tests.md b/website/content/en/docs/reference/configuration/unit-tests.md index 5e11240c9136f..d470813252518 100644 --- a/website/content/en/docs/reference/configuration/unit-tests.md +++ b/website/content/en/docs/reference/configuration/unit-tests.md @@ -77,12 +77,11 @@ functions][type], functions like [`exists`][exists], [`includes`][includes], [`is_nullish`][is_nullish] and [`contains`][contains], and VRL [comparisons]. Here's an example usage of a Boolean expression passed to an `assert` function: -```toml -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert!(is_string(.message) && is_timestamp(.timestamp) && !exists(.other)) -''' +```yaml +conditions: + - type: "vrl" + source: | + assert!(is_string(.message) && is_timestamp(.timestamp) && !exists(.other)) ``` In this case, the VRL program (under `source`) evaluates to a single Boolean that expresses the @@ -94,29 +93,27 @@ following: It's also possible to break a test up into multiple `assert` or `assert_eq` statements: -```toml -source = ''' -assert!(exists(.message), "no message field provided") -assert!(!is_nullish(.message), "message field is an empty string") -assert!(is_string(.message), "message field has as unexpected type") -assert_eq!(.message, "success", "message field had an unexpected value") -assert!(exists(.timestamp), "no timestamp provided") -assert!(is_timestamp(.timestamp), "timestamp is invalid") -assert!(!exists(.other), "extraneous other field present") -''' +```yaml +source: | + assert!(exists(.message), "no message field provided") + assert!(!is_nullish(.message), "message field is an empty string") + assert!(is_string(.message), "message field has as unexpected type") + assert_eq!(.message, "success", "message field had an unexpected value") + assert!(exists(.timestamp), "no timestamp provided") + assert!(is_timestamp(.timestamp), "timestamp is invalid") + assert!(!exists(.other), "extraneous other field present") ``` You can also store the Boolean expressions in variables rather than passing the entire statement to the `assert` function: -```toml -source = ''' -message_field_valid = exists(.message) && - !is_nullish(.message) && - .message == "success" +```yaml +source: | + message_field_valid = exists(.message) && + !is_nullish(.message) && + .message == "success" -assert!(message_field_valid) -''' + assert!(message_field_valid) ``` ## Example unit test configuration {#example} @@ -124,48 +121,48 @@ assert!(message_field_valid) Below is an annotated example of a unit test suite for a transform called `add_metadata`, which adds a unique ID and a timestamp to log events: -```toml -[sources.all_container_services] -type = "docker_logs" -docker_host = "http://localhost:2375" -include_images = ["web_frontend", "web_backend", "auth_service"] +```yaml +sources: + all_container_services: + type: "docker_logs" + docker_host: "http://localhost:2375" + include_images: ["web_frontend", "web_backend", "auth_service"] # The transform being tested is a Vector Remap Language (VRL) transform that # adds two fields to each incoming log event: a timestamp and a unique ID -[transforms.add_metadata] -type = "remap" -inputs = ["all_container_services"] -source = ''' -.timestamp = now() -.id = uuid_v4() -''' +transforms: + add_metadata: + type: "remap" + inputs: ["all_container_services"] + source: | + .timestamp = now() + .id = uuid_v4() # Here we begin configuring our test suite -[[tests]] -name = "Test for the add_metadata transform" - -# The inputs for the test -[[tests.inputs]] -insert_at = "add_metadata" # The transform into which the testing event is inserted -type = "log" # The event type (either log or metric) - -# The test log event that is passed to the `add_metadata` transform -[tests.inputs.log_fields] -message = "successful transaction" -code = 200 - -# The expected outputs of the test -[[tests.outputs]] -extract_from = "add_metadata" # The transform from which the resulting event is extracted - -# The declaration of what we expect -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert!(is_timestamp(.timestamp)) -assert!(is_string(.id)) -assert_eq!(.message, "successful transaction") -''' +tests: + - name: "Test for the add_metadata transform" + + # The inputs for the test + inputs: + - insert_at: "add_metadata" # The transform into which the testing event is inserted + type: "log" # The event type (either log or metric) + + # The test log event that is passed to the `add_metadata` transform + log_fields: + message: "successful transaction" + code: 200 + + # The expected outputs of the test + outputs: + - extract_from: "add_metadata" # The transform from which the resulting event is extracted + + # The declaration of what we expect + conditions: + - type: "vrl" + source: | + assert!(is_timestamp(.timestamp)) + assert!(is_string(.id)) + assert_eq!(.message, "successful transaction") ``` This example represents a complete test of the `add_metadata` transform, include test `inputs` @@ -187,7 +184,7 @@ unit tests as a way of **mocking observability data sources** and ensuring that respond to those mock sources the way that you would expect. {{< success title="Multiple config formats available" >}} -The unit testing example above is in TOML but Vector also supports YAML and JSON as configuration +The unit testing example above is in YAML but Vector also supports TOML and JSON as configuration formats. {{< /success >}} @@ -198,16 +195,15 @@ same config file alongside your transform definitions or split them out into a s Unit tests need are specified inside of a `tests` array. Each test requires a `name`: -```toml -[[tests]] -name = "test 1" -# Other test config +```yaml +tests: + - name: "test 1" + # Other test config -[[tests]] -name = "test_2" -# Other test config + - name: "test_2" + # Other test config -# etc. + # etc. ``` Inside each test definition, you need to specify two things: @@ -230,18 +226,19 @@ In the `inputs` array for the test, you have these options: Here's an example `inputs` declaration: -```toml -[transforms.add_metadata] -# transform config +```yaml +transforms: + add_metadata: + # transform config -[[tests]] -name = "Test add_metadata transform" +tests: + - name: "Test add_metadata transform" -[[tests.inputs]] -insert_at = "add_metadata" + inputs: + - insert_at: "add_metadata" -[tests.inputs.log_fields] -message = "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" + log_fields: + message: "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" ``` ### Outputs @@ -262,16 +259,15 @@ Each condition in the `conditions` array has two fields: Here's an example `outputs` declaration: -```toml -[[tests.outputs]] -extract_from = "add_metadata" - -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert!(is_string(.id)) -assert!(exists(.tags)) -''' +```yaml +outputs: + - extract_from: "add_metadata" + + conditions: + - type: "vrl" + source: | + assert!(is_string(.id)) + assert!(exists(.tags)) ``` #### Asserting no output @@ -280,10 +276,10 @@ In some cases, you may need to assert that _no_ event is output by a transform. this at the root level of a specific test's configuration using the `no_outputs_from` parameter, which takes a list of transform names. Here's an example: -```toml -[[tests]] -name = "Ensure no output" -no_outputs_from = ["log_filter", "metric_filter"] +```yaml +tests: + - name: "Ensure no output" + no_outputs_from: ["log_filter", "metric_filter"] ``` In this test configuration, Vector would expect that the `log_filter` and `metric_filter` transforms @@ -298,26 +294,27 @@ Some examples of use cases for `no_outputs_from`: Below is a full example of using `no_outputs_from` in a Vector unit test: -```toml -[transforms.log_filter] -type = "filter" -inputs = ["log_source"] -condition = '.env == "production"' - -[[tests]] -name = "Filter out non-production events" -no_outputs_from = ["log_filter"] - -[[tests.inputs]] -type = "log" -insert_at = "log_filter" - -[tests.inputs.log_fields] -message = "success" -code = 202 -endpoint = "/transactions" -method = "POST" -env = "staging" +```yaml +transforms: + log_filter: + type: "filter" + inputs: ["log_source"] + condition: '.env == "production"' + +tests: + - name: "Filter out non-production events" + no_outputs_from: ["log_filter"] + + inputs: + - type: "log" + insert_at: "log_filter" + + log_fields: + message: "success" + code: 202 + endpoint: "/transactions" + method: "POST" + env: "staging" ``` This unit test passes because the `env` field of the input event has a value of `staging`, which @@ -340,11 +337,11 @@ either a structured event [object](#object) or a raw [string](#raw-string-value) To specify a structured log event as your test input, use `log_fields`: -```toml -[tests.inputs.log_fields] -message = "successful transaction" -code = 200 -id = "38c5b0d0-5e7e-42aa-ae86-2b642ad2d1b8" +```yaml +log_fields: + message: "successful transaction" + code: 200 + id: "38c5b0d0-5e7e-42aa-ae86-2b642ad2d1b8" ``` If there are hyphens in the field name, you will need to quote this part (at least in YAML): @@ -362,38 +359,38 @@ If there are hyphens in the field name, you will need to quote this part (at lea To specify a raw string value for a log event, use `value`: -```toml -[[tests.inputs]] -insert_at = "add_metadata" -value = "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" +```yaml +inputs: + - insert_at: "add_metadata" + value: "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" ``` ##### VRL program To specify a program to construct the log event, use `source`: -```toml -[[tests.inputs]] - insert_at = "canary" - type = "vrl" - source = """ - . = {"a": {"b": "c"}, "d": now()} - """ +```yaml +inputs: + - insert_at: "canary" + type: "vrl" + source: | + . = {"a": {"b": "c"}, "d": now()} ``` #### Metrics You can specify the fields in a metric event to be unit tested using a `metric` object: -```toml -[[tests.inputs]] -insert_at = "my_metric_transform" -type = "metric" +```yaml +inputs: + - insert_at: "my_metric_transform" + type: "metric" -[tests.inputs.metric] -name = "count" -kind = "absolute" -counter = { value = 1 } + metric: + name: "count" + kind: "absolute" + counter: + value: 1 ``` Aggregated metrics are a little different: @@ -414,40 +411,40 @@ tests: Here's a full end-to-end example of unit testing a metric through a transform: -```toml -[transforms.add_env_to_metric] -type = "remap" -inputs = [] -source = ''' -env, err = get_env_var("ENV") -if err != null { - log(err, level: "error") -} -tags.environment = env -''' - -[[tests]] -name = "add_unique_id_test" - -[[tests.inputs]] -insert_at = "add_env_to_metric" -type = "metric" - -[tests.inputs.metric] -name = "website_hits" -kind = "absolute" -counter = { value = 1 } - -[[tests.outputs]] -extract_from = "add_env_to_metric" - -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert_eq!(.name, "website_hits") -assert_eq!(.kind, "absolute") -assert_eq!(.tags.environment, "production") -''' +```yaml +transforms: + add_env_to_metric: + type: "remap" + inputs: [] + source: | + env, err = get_env_var("ENV") + if err != null { + log(err, level: "error") + } + tags.environment = env + +tests: + - name: "add_unique_id_test" + + inputs: + - insert_at: "add_env_to_metric" + type: "metric" + + metric: + name: "website_hits" + kind: "absolute" + counter: + value: 1 + + outputs: + - extract_from: "add_env_to_metric" + + conditions: + - type: "vrl" + source: | + assert_eq!(.name, "website_hits") + assert_eq!(.kind, "absolute") + assert_eq!(.tags.environment, "production") ``` ## Multiple transforms {#multiple} @@ -464,73 +461,71 @@ You may notice that the three transforms in this example could be combined into transform. Their separation into multiple transforms here is purely for demonstration purposes. {{< /warning >}} -```toml +```yaml # This source, like all sources, is ignored in the unit test itself -[sources.web_backend] -type = "docker_logs" -docker_host = "http://localhost:2375" -include_images = ["web_backend"] +sources: + web_backend: + type: "docker_logs" + docker_host: "http://localhost:2375" + include_images: ["web_backend"] # The first transform in the chain -[transforms.add_env_metadata] -type = "remap" -inputs = ["web_backend"] -source = ''' -.tags.environment = "production" -''' - -# The second transform in the chain -[transforms.sanitize] -type = "remap" -inputs = ["add_env_metadata"] -source = ''' -del(.username) -del(.email) -''' - -# The final transform in the chain -[transforms.add_host_metadata] -type = "remap" -inputs = ["sanitize"] -source = ''' -.tags.host = "web-backend1.vector-user.biz" -''' - -[[tests]] -name = "Multiple chained remap transforms" - -[[tests.inputs]] -type = "log" -# Insert test input events into the first transform -insert_at = "add_env_metadata" - -# The input event to insert into the first transform in the chain -[tests.inputs.log_fields] -message = "image successfully uploaded" -code = 202 -username = "tonydanza1337" -email = "tony@whostheboss.com" -transaction_id = "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" - -[[tests.outputs]] -# Extract test outputs from the last transform -extract_from = "add_host_metadata" - -[[tests.outputs.conditions]] -type = "vrl" -# Our VRL assertions for the test output -source = ''' -assert_eq!(.tags.environment, "production", "incorrect environment tag") -assert_eq!(.tags.host, "web-backend1.vector-user.biz", "incorrect host tag") -assert!(!exists(.username)) -assert!(!exists(.email)) - -valid_transaction_id = exists(.transaction_id) && - is_string(.transaction_id) && - length!(.transaction_id) == 36 - -assert!(valid_transaction_id, "transaction ID invalid") -''' +transforms: + add_env_metadata: + type: "remap" + inputs: ["web_backend"] + source: | + .tags.environment = "production" + + # The second transform in the chain + sanitize: + type: "remap" + inputs: ["add_env_metadata"] + source: | + del(.username) + del(.email) + + # The final transform in the chain + add_host_metadata: + type: "remap" + inputs: ["sanitize"] + source: | + .tags.host = "web-backend1.vector-user.biz" + +tests: + - name: "Multiple chained remap transforms" + + inputs: + - type: "log" + # Insert test input events into the first transform + insert_at: "add_env_metadata" + + # The input event to insert into the first transform in the chain + log_fields: + message: "image successfully uploaded" + code: 202 + username: "tonydanza1337" + email: "tony@whostheboss.com" + transaction_id: "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" + + outputs: + - # Extract test outputs from the last transform + extract_from: "add_host_metadata" + + conditions: + - type: "vrl" + # Our VRL assertions for the test output + source: | + assert_eq!(.tags.environment, "production", "incorrect environment tag") + assert_eq!(.tags.host, "web-backend1.vector-user.biz", "incorrect host tag") + assert!(!exists(.username)) + assert!(!exists(.email)) + + valid_transaction_id = exists(.transaction_id) && + is_string(.transaction_id) && + length!(.transaction_id) == 36 + + assert!(valid_transaction_id, "transaction ID invalid") ``` From a testing standpoint, all three transforms here can be thought of as a single unit. One example @@ -541,41 +536,40 @@ extracted from the end of the chain (`add_host_metadata`), and one set of VRL You could also test a subset of this transform chain. This configuration, for example, would test only the first two transforms (`add_env_metadata` and `sanitize`): -```toml -[[tests]] -name = "First two transforms" - -[[tests.inputs]] -type = "log" -# Insert test input into the first transform -insert_at = "add_env_metadata" - -# For comparison, we can use the same input event as above -[tests.inputs.log_fields] -message = "image successfully uploaded" -code = 202 -username = "tonydanza1337" -email = "tony@whostheboss.com" -transaction_id = "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" - -[[tests.outputs]] -# Extract test output from the second transform rather than the last -extract_from = "sanitize" - -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert_eq!(.tags.environment, "production", "incorrect environment tag") -assert!(!exists(.tags.host), "host tag included") -assert!(!exists(.username)) -assert!(!exists(.email)) +```yaml +tests: + - name: "First two transforms" -valid_transaction_id = exists(.transaction_id) && - is_string(.transaction_id) && - length!(.transaction_id) == 36 + inputs: + - type: "log" + # Insert test input into the first transform + insert_at: "add_env_metadata" -assert!(valid_transaction_id, "transaction ID invalid") -''' + # For comparison, we can use the same input event as above + log_fields: + message: "image successfully uploaded" + code: 202 + username: "tonydanza1337" + email: "tony@whostheboss.com" + transaction_id: "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" + + outputs: + - # Extract test output from the second transform rather than the last + extract_from: "sanitize" + + conditions: + - type: "vrl" + source: | + assert_eq!(.tags.environment, "production", "incorrect environment tag") + assert!(!exists(.tags.host), "host tag included") + assert!(!exists(.username)) + assert!(!exists(.email)) + + valid_transaction_id = exists(.transaction_id) && + is_string(.transaction_id) && + length!(.transaction_id) == 36 + + assert!(valid_transaction_id, "transaction ID invalid") ``` In the VRL conditions for this two-transform test, notice that the assertion regarding the `host` diff --git a/website/content/en/docs/setup/going-to-prod/high-availability.md b/website/content/en/docs/setup/going-to-prod/high-availability.md index 768d40d16ea59..7fa71f52291a2 100644 --- a/website/content/en/docs/setup/going-to-prod/high-availability.md +++ b/website/content/en/docs/setup/going-to-prod/high-availability.md @@ -53,23 +53,26 @@ We recommend routing dropped events immediately to a backup destination to prior The following is an example configuration that routes failed events from a `remap` transform to a `aws_s3` sink: -```toml -[sources.input] - type = "datadog_agent" - -[transforms.parsing] - inputs = ["input"] - type = "remap" - reroute_dropped = true - source = "..." - -[sinks.analysis] - inputs = ["parsing"] - type = "datadog_logs" - -[sinks.backup] - inputs = ["**parsing.dropped**"] # dropped events from the `parsing` transform - type = "aws_s3" +```yaml +sources: + input: + type: "datadog_agent" + +transforms: + parsing: + inputs: ["input"] + type: "remap" + reroute_dropped: true + source: "..." + +sinks: + analysis: + inputs: ["parsing"] + type: "datadog_logs" + + backup: + inputs: ["parsing.dropped"] # dropped events from the `parsing` transform + type: "aws_s3" ``` {{< info >}} diff --git a/website/content/en/guides/advanced/custom-aggregations-with-lua.md b/website/content/en/guides/advanced/custom-aggregations-with-lua.md index 262f6b7a2d4c8..4cfc4578b2343 100644 --- a/website/content/en/guides/advanced/custom-aggregations-with-lua.md +++ b/website/content/en/guides/advanced/custom-aggregations-with-lua.md @@ -31,12 +31,12 @@ There are three types of hooks: `init`, `process`, and `shutdown`. The most important of them is `process`, which is called on each incoming events. It can be defined like this: -```toml -hooks.process = """ -function (event, emit) - -- do something -end -""" +```yaml +hooks: + process: | + function (event, emit) + -- do something + end ``` It takes a single event and can output one or many of them using the `emit` function provided as the second argument. @@ -76,54 +76,54 @@ variables. Using the knowledge from the previous section, it is possible to write down the following transform definition: -```toml title="vector.toml" -[transforms.aggregator] -type = "lua" -version = "2" -inputs = [] # add IDs of the input components here - -hooks.init = """ - function (emit) - count = 0 -- initialize state by setting a global variable - end -""" - -hooks.process = """ - function (event, emit) - count = count + 1 -- increment the counter and exit - end -""" - -timers = [{interval_seconds = 5, handler = """ - function (emit) - emit { - metric = { - name = "event_counter", - kind = "incremental", - timestamp = os.date("!*t"), - counter = { - value = counter - } - } - } - counter = 0 - end -"""}] - -hooks.shutdown = """ - function (emit) - emit { - metric = { - name = "event_counter", - kind = "incremental", - timestamp = os.date("!*t"), - counter = { - value = counter - } - } - } - end -""" +```yaml title="vector.yaml" +transforms: + aggregator: + type: "lua" + version: "2" + inputs: [] # add IDs of the input components here + + hooks: + init: | + function (emit) + count = 0 -- initialize state by setting a global variable + end + + process: | + function (event, emit) + count = count + 1 -- increment the counter and exit + end + + shutdown: | + function (emit) + emit { + metric = { + name = "event_counter", + kind = "incremental", + timestamp = os.date("!*t"), + counter = { + value = counter + } + } + } + end + + timers: + - interval_seconds: 5 + handler: | + function (emit) + emit { + metric = { + name = "event_counter", + kind = "incremental", + timestamp = os.date("!*t"), + counter = { + value = counter + } + } + } + counter = 0 + end ``` One could plug it into a [pipeline][docs.about.concepts#pipelines] and it would work! @@ -137,8 +137,8 @@ identical. It is possible make the config more [DRY][urls.dry_code] by extractin a dedicated function. Such a function can be placed into the [source][docs.transforms.lua#source] section of the config: -```toml -source = """ +```yaml +source: | function make_counter(value) return metric = { name = "event_counter", @@ -149,28 +149,28 @@ source = """ } } end -""" ``` and then adjusting the timer handler -```toml -timers = [{interval_seconds = 5, handler = """ - function (emit) - emit(make_counter(counter)) - counter = 0 - end -"""}] +```yaml +timers: + - interval_seconds: 5 + handler: | + function (emit) + emit(make_counter(counter)) + counter = 0 + end ``` and the `shutdown` hook: -```toml -hooks.shutdown = """ - function (emit) - emit(make_counter(counter)) - end -""" +```yaml +hooks: + shutdown: | + function (emit) + emit(make_counter(counter)) + end ``` ## Keep All Code Together @@ -178,45 +178,48 @@ hooks.shutdown = """ The new config looks tidier, but in order to make it more readable, it is also possible to gather implementations of all functions into the `source` section, resulting in the following component declaration: -```toml title="vector.toml" -[transforms.aggregator] -type = "lua" -version = "2" -inputs = [] # add IDs of the input components here -hooks.init = "init" -hooks.process = "process" -hooks.shutdown = "shutdown" -timers = [{interval_seconds = 5, handler = "timer_handler"}] - -source = """ - function init() - count = 0 - end - - function process() - count = count + 1 - end - - function timer_handler(emit) - emit(make_counter(counter)) - counter = 0 - end - - function shutdown(emit) - emit(make_counter(counter)) - end - - function make_counter(value) - return metric = { - name = "event_counter", - kind = "incremental", - timestamp = os.date("!*t"), - counter = { - value = value - } - } - end -""" +```yaml title="vector.yaml" +transforms: + aggregator: + type: "lua" + version: "2" + inputs: [] # add IDs of the input components here + hooks: + init: "init" + process: "process" + shutdown: "shutdown" + timers: + - interval_seconds: 5 + handler: "timer_handler" + + source: | + function init() + count = 0 + end + + function process() + count = count + 1 + end + + function timer_handler(emit) + emit(make_counter(counter)) + counter = 0 + end + + function shutdown(emit) + emit(make_counter(counter)) + end + + function make_counter(value) + return metric = { + name = "event_counter", + kind = "incremental", + timestamp = os.date("!*t"), + counter = { + value = value + } + } + end ``` ## A Loadable Module @@ -262,16 +265,20 @@ end and -```toml title="vector.toml" -[transforms.aggregator] -type = "lua" -version = "2" -inputs = [] # add IDs of the input components here -hooks.init = "init" -hooks.process = "process" -hooks.shutdown = "shutdown" -timers = [{interval_seconds = 5, handler = "timer_handler"}] -source = "require('aggregator')" +```yaml title="vector.yaml" +transforms: + aggregator: + type: "lua" + version: "2" + inputs: [] # add IDs of the input components here + hooks: + init: "init" + process: "process" + shutdown: "shutdown" + timers: + - interval_seconds: 5 + handler: "timer_handler" + source: "require('aggregator')" ``` There are also [other possibilities][urls.lua_modules_tutorial] to define Lua modules which do not require to use diff --git a/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md b/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md index c90fa50527457..45dd47ea4c7ff 100644 --- a/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md +++ b/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md @@ -43,44 +43,44 @@ in the [guide on parsing CSV logs][guides.parsing-csv-logs-with-lua]. The underl Such an algorithm can be implemented, for example, with the following transform config: -```toml title="vector.toml" -[transforms.lua] - inputs = [] - type = "lua" - version = "2" - source = """ - csv = require("csv") -- load the `lua-csv` module - expected_columns = 23 -- expected number of columns in incoming CSV lines - line_separator = "\\r\\n" -- note the double escaping required by the TOML format - """ - hooks.process = """ - function (event, emit) - merged_event = merge(event) - if merged_event == nil then -- a global variable containing the merged event - merged_event = event -- if it is empty, set it to the current event - else -- otherwise, concatenate the line in the stored merged event - -- with the next line - merged_event.log.message = merged_event.log.message .. - line_separator .. event.log.message - end - - fields = csv.openstring(event.log.message):lines()() -- parse CSV - if #fields < expected_columns then - return -- not all fields are present in the merged event yet - end - - -- do something with the array of the parsed fields - merged_event.log.csv_fields = fields -- for example, just store them in an - -- array field - - emit(merged_event) -- emit the resulting event - merged_event = nil -- clear the merged event - end - """ +```yaml title="vector.yaml" +transforms: + lua: + inputs: [] + type: "lua" + version: "2" + source: | + csv = require("csv") -- load the `lua-csv` module + expected_columns = 23 -- expected number of columns in incoming CSV lines + line_separator = "\r\n" + hooks: + process: | + function (event, emit) + merged_event = merge(event) + if merged_event == nil then -- a global variable containing the merged event + merged_event = event -- if it is empty, set it to the current event + else -- otherwise, concatenate the line in the stored merged event + -- with the next line + merged_event.log.message = merged_event.log.message .. + line_separator .. event.log.message + end + + fields = csv.openstring(event.log.message):lines()() -- parse CSV + if #fields < expected_columns then + return -- not all fields are present in the merged event yet + end + + -- do something with the array of the parsed fields + merged_event.log.csv_fields = fields -- for example, just store them in an + -- array field + + emit(merged_event) -- emit the resulting event + merged_event = nil -- clear the merged event + end ``` -In this code sample, the `source` option defines code that's executed when the transform is created. -while the `hooks.process` option defines a function that's called for each incoming event. +In this code sample, the `source` option defines code that is executed when the transform is created, +while the `hooks.process` option defines a function that is called for each incoming event. ## How It Works diff --git a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md index 00123d1ace5d0..634b04f7e3006 100644 --- a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md +++ b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md @@ -36,36 +36,40 @@ log file: Let us draft an initial version of the Vector's configuration file: -```toml title="vector.toml" -data_dir = "." - -[sources.file] - type = "file" - include = ["*.csv"] - ignore_checkpoints = true - -[transforms.lua] - inputs = ["file"] - type = "lua" - version = "2" - hooks.process = """ - function (event, emit) - -- to be expanded - emit(event) - end - """ - -[sinks.console] - inputs = ["lua"] - type = "console" - encoding.codec = "json" +```yaml title="vector.yaml" +data_dir: "." + +sources: + file: + type: "file" + include: ["*.csv"] + ignore_checkpoints: true + +transforms: + lua: + inputs: ["file"] + type: "lua" + version: "2" + hooks: + process: | + function (event, emit) + -- to be expanded + emit(event) + end + +sinks: + console: + inputs: ["lua"] + type: "console" + encoding: + codec: "json" ``` This config sets up a [pipeline][docs.meta.glossary#pipeline] that reads log files, pipes them through the parsing transform (which currently is configured to just pass the events through), and displays the produced log events using [`console`][docs.sinks.console] sink. -At this point, running `vector --config vector.toml` results in the following output: +At this point, running `vector --config vector.yaml` results in the following output: ```json {"file":"log.csv","host":"localhost","message":"2020-04-09 12:48:49.661 UTC,,,1,,localhost.1,1,,2020-04-09 12:48:49 UTC,,0,LOG,00000,\"ending log output to stderr\",,\"Future log output will go to log destination \"\"csvlog\"\".\",,,,,,,\"\"","timestamp":"2020-04-09T14:33:28Z"} @@ -77,7 +81,7 @@ At this point, running `vector --config vector.toml` results in the following ou In order to perform actual parsing, it is possible to leverage [`lua-csv`][urls.lua_csv_repo]. Because it consists of a [single file][urls.lua_csv_view], it is possible to just download it to the same -directory where `vector.toml` is stored: +directory where `vector.yaml` is stored: ```bash curl -o csv.lua https://raw.githubusercontent.com/geoffleyland/lua-csv/d20cd42d61dc52e7f6bcb13b596ac7a7d4282fbf/lua/csv.lua @@ -86,10 +90,9 @@ curl -o csv.lua https://raw.githubusercontent.com/geoffleyland/lua-csv/d20cd42d6 Then it would be possible to load it by calling [`require`][urls.lua_require] Lua function in the [`source`][docs.transforms.lua#source] configuration section: -```toml -source = """ +```yaml +source: | csv = require("csv") -""" ``` With this `source` the `csv` module is loaded when Vector is started up (or if the `lua` transform is added later and the @@ -99,85 +102,85 @@ config is automatically reloaded) and can be used through the global variable `c With the `csv` module, the [`hooks.process`][docs.transforms.lua#process] can be changed to the following: -```toml -hooks.process = """ - function (event, emit) - fields = csv.openstring(event.log.message):lines()() -- parse the `message` field - event.log.message = nil -- drop the `message` field +```yaml +hooks: + process: | + function (event, emit) + fields = csv.openstring(event.log.message):lines()() -- parse the `message` field + event.log.message = nil -- drop the `message` field - column_names = { -- a sequence containing CSV column names - -- ... - } + column_names = { -- a sequence containing CSV column names + -- ... + } - for column, value in ipairs(fields) do -- iterate over CSV columns - column_name = column_names[column] -- get column name - event.log[column_name] = value -- set the corresponding field in the event - end + for column, value in ipairs(fields) do -- iterate over CSV columns + column_name = column_names[column] -- get column name + event.log[column_name] = value -- set the corresponding field in the event + end - emit(event) -- emit the transformed event - end -""" + emit(event) -- emit the transformed event + end ``` Note that the `column_names` can be created just once, in the `source` section instead to speed up processing. Putting it there and using the column names from the PostgreSQL documentation results in the following definition of the whole transform: -```toml title="vector.toml" +```yaml title="vector.yaml" +# ... +transforms: + lua: + inputs: ["file"] + type: "lua" + version: "2" + source: | + csv = require("csv") -- load external module for parsing CSV + column_names = { -- a sequence containing CSV column names + "log_time", + "user_name", + "database_name", + "process_id", + "connection_from", + "session_id", + "session_line_num", + "command_tag", + "session_start_time", + "virtual_transaction_id", + "transaction_id", + "error_severity", + "sql_state_code", + "message", + "detail", + "hint", + "internal_query", + "internal_query_pos", + "context", + "query", + "query_pos", + "location", + "application_name", + -- available only in postgres > 13, to remove for postgres <= 13 + "backend_type", + "leader_pid", + "query_id" + } + hooks: + process: | + function (event, emit) + fields = csv.openstring(event.log.message):lines()() -- parse the `message` field + event.log.message = nil -- drop the `message` field + + for column, column_name in ipairs(column_names) do -- iterate over column names + value = fields[column] -- get field value + event.log[column_name] = value -- set the corresponding field in the event + end + + emit(event) -- emit the transformed event + end # ... -[transforms.lua] - inputs = ["file"] - type = "lua" - version = "2" - source = """ - csv = require("csv") -- load external module for parsing CSV - column_names = { -- a sequence containing CSV column names - "log_time", - "user_name", - "database_name", - "process_id", - "connection_from", - "session_id", - "session_line_num", - "command_tag", - "session_start_time", - "virtual_transaction_id", - "transaction_id", - "error_severity", - "sql_state_code", - "message", - "detail", - "hint", - "internal_query", - "internal_query_pos", - "context", - "query", - "query_pos", - "location", - "application_name", - -- available only in postgres > 13, to remove for postgres <= 13 - "backend_type", - "leader_pid", - "query_id" - } - """ - hooks.process = """ - function (event, emit) - fields = csv.openstring(event.log.message):lines()() -- parse the `message` field - event.log.message = nil -- drop the `message` field - - for column, column_name in ipairs(column_names) do -- iterate over column names - value = fields[column] -- get field value - event.log[column_name] = value -- set the corresponding field in the event - end - - emit(event) -- emit the transformed event - end - """ -#... ``` -Trying to run `vector --config vector.toml` with the same input file results in structured events being output: +Trying to run `vector --config vector.yaml` with the same input file results in structured events being output: ```json {"application_name":"","backend_type":"not initialized","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"Future log output will go to log destination \"csvlog\".","host":"localhost","internal_query":"","internal_query_pos":"","leader_pid":"","location":"","log_time":"2020-04-09 12:48:49.661 UTC","message":"ending log output to stderr","process_id":"1","query":"","query_id":"0","query_pos":"","session_id":"localhost.1","session_line_num":"1","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} diff --git a/website/content/en/guides/aws/cloudwatch-logs-firehose.md b/website/content/en/guides/aws/cloudwatch-logs-firehose.md index 02c4e48b40146..0760a1e2d70e8 100644 --- a/website/content/en/guides/aws/cloudwatch-logs-firehose.md +++ b/website/content/en/guides/aws/cloudwatch-logs-firehose.md @@ -79,33 +79,36 @@ export FIREHOSE_S3_BUCKET="firehose-${AWS_ACCOUNT_ID}" # a bucket to write event Let's take a look at the configuration we will be using: -```toml title="vector.toml" -[sources.firehose] - type = "aws_kinesis_firehose" - address = "0.0.0.0:8080" # the public URL will be set when configuring Firehose - access_key = "${FIREHOSE_ACCESS_KEY} # this will also be set when configuring Firehose - -[transforms.parse] - type = "remap" - inputs = ["firehose"] - drop_on_error = false - source = ''' - parsed = parse_aws_cloudwatch_log_subscription_message!(.message) - . = unnest(parsed.log_events) - . = map_values(.) -> |value| { - event = del(value.log_events) - value |= event - message = del(.message) - . |= object!(parse_json!(message)) - } - ''' +```yaml title="vector.yaml" +sources: + firehose: + type: "aws_kinesis_firehose" + address: "0.0.0.0:8080" # the public URL will be set when configuring Firehose + access_key: "${FIREHOSE_ACCESS_KEY}" # this will also be set when configuring Firehose + +transforms: + parse: + type: "remap" + inputs: ["firehose"] + drop_on_error: false + source: | + parsed = parse_aws_cloudwatch_log_subscription_message!(.message) + . = unnest(parsed.log_events) + . = map_values(.) -> |value| { + event = del(value.log_events) + value |= event + message = del(.message) + . |= object!(parse_json!(message)) + } # you may want to add more transforms here -[sinks.console] - type = "console" - inputs = ["parse"] - encoding.codec = "json" +sinks: + console: + type: "console" + inputs: ["parse"] + encoding: + codec: "json" ``` This will configure `vector` to listen for Firehose messages on the configured @@ -352,16 +355,19 @@ delivery stream. To make sure everything is wired up correctly, let's send some logs to the log group we've setup. We can use Vector for this too! -```bash -[sources.stdin] - type = "stdin" -[sinks.cloudwatch] - type = "aws_cloudwatch_logs" - inputs = ["stdin"] - group_name = "${LOG_GROUP}" - stream_name = "test" - region = "us-east-1" - encoding.codec = "json" +```yaml +sources: + stdin: + type: "stdin" +sinks: + cloudwatch: + type: "aws_cloudwatch_logs" + inputs: ["stdin"] + group_name: "${LOG_GROUP}" + stream_name: "test" + region: "us-east-1" + encoding: + codec: "json" ``` This will read lines from `stdin` and write them to CloudWatch Logs. See the @@ -377,7 +383,7 @@ Let's send some logs. For this, I'm using log data. ```bash -flog -f json | vector --config config.toml +flog -f json | vector --config config.yaml ``` This will send some logs to your log group. Within 300 seconds (the default @@ -450,11 +456,12 @@ send it as an HTTP request: The [`aws_kinesis_firehose`][aws_kinesis_firehose] source: -```toml -[sources.firehose] - type = "aws_kinesis_firehose" - address = "0.0.0.0:8080" # the public URL will be set in the Firehose config - access_key = "my secret key" # this will also be set in the Firehose config +```yaml +sources: + firehose: + type: "aws_kinesis_firehose" + address: "0.0.0.0:8080" # the public URL will be set in the Firehose config + access_key: "my secret key" # this will also be set in the Firehose config ``` will accept this request, decode the record (which is gzip'ed and then base64 encoded), to produce an event that looks like: @@ -479,21 +486,21 @@ can use a [`remap`][remap] transform and leverage the [`parse_aws_cloudwatch_log [`unnest`][unnest], [`map_values`][map_values], and [`parse_json`][parse_json] functions: -```toml -[transforms.parse] - type = "remap" - inputs = ["firehose"] - drop_on_error = false - source = ''' - parsed = parse_aws_cloudwatch_log_subscription_message!(.message) - . = unnest(parsed.log_events) - . = map_values(.) -> |value| { - event = del(value.log_events) - value |= event - message = del(.message) - . |= object!(parse_json!(message)) - } - ''' +```yaml +transforms: + parse: + type: "remap" + inputs: ["firehose"] + drop_on_error: false + source: | + parsed = parse_aws_cloudwatch_log_subscription_message!(.message) + . = unnest(parsed.log_events) + . = map_values(.) -> |value| { + event = del(value.log_events) + value |= event + message = del(.message) + . |= object!(parse_json!(message)) + } ``` Let's step through this program one function at a time. diff --git a/website/content/en/guides/level-up/managing-complex-configs.md b/website/content/en/guides/level-up/managing-complex-configs.md index 5f55a70be97ae..88c7a58a93923 100644 --- a/website/content/en/guides/level-up/managing-complex-configs.md +++ b/website/content/en/guides/level-up/managing-complex-configs.md @@ -40,28 +40,32 @@ For example, if we wished to create a chain of three transforms; `remap`, `filte and `reduce`, we can run: ```bash -vector generate /remap,filter,reduce > vector.toml +vector generate /remap,filter,reduce > vector.yaml # Find out more with `vector generate --help` ``` And most of the boilerplate will be written for us, with each component printed with an `inputs` field that specifies the component before it: -```toml title="vector.toml" -[transforms.transform0] - inputs = [ "somewhere" ] - type = "remap" - # etc ... - -[transforms.transform1] - inputs = [ "transform0" ] - type = "filter" - # etc ... - -[transforms.transform2] - inputs = [ "transform1" ] - type = "reduce" - # etc ... +```yaml title="vector.yaml" +transforms: + transform0: + inputs: + - "somewhere" + type: "remap" + # etc ... + + transform1: + inputs: + - "transform0" + type: "filter" + # etc ... + + transform2: + inputs: + - "transform1" + type: "reduce" + # etc ... ``` The IDs of the generated components are sequential (`transform0`, @@ -79,18 +83,20 @@ Let's imagine we are in the process of building the config from the [unit test guide][guides.unit-testing], we might start off with our source and the grok parser: -```toml title="vector.toml" -[sources.over_tcp] - type = "socket" - mode = "tcp" - address = "0.0.0.0:9000" - -[transforms.foo] - inputs = ["over_tcp"] - type = "remap" - source = ''' - . = parse_grok!(.message, s'%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}') -''' +```yaml title="vector.yaml" +sources: + over_tcp: + type: "socket" + mode: "tcp" + address: "0.0.0.0:9000" + +transforms: + foo: + inputs: + - "over_tcp" + type: "remap" + source: | + . = parse_grok!(.message, s'%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}') ``` A common way to test this transform might be to temporarily change the source @@ -101,30 +107,28 @@ our config to run tests rather than focusing on features. Instead, we can leave our source as a `socket` type and add a unit test to the end of our config: -```toml title="vector.toml" -[[tests]] - name = "check_simple_log" - - [[tests.inputs]] - insert_at = "foo" - type = "raw" - value = "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" - - [[tests.outputs]] - extract_from = "foo" +```yaml title="vector.yaml" +tests: + - name: "check_simple_log" + inputs: + - insert_at: "foo" + type: "raw" + value: "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" + outputs: + - extract_from: "foo" ``` When we add a unit test output without any conditions it will simply print the input and output events of a transform, allowing us to inspect its behavior: ```sh -$ vector test ./vector.toml -Running vector.toml tests -test vector.toml: check_simple_log ... passed +$ vector test ./vector.yaml +Running vector.yaml tests +test vector.yaml: check_simple_log ... passed inspections: ---- vector.toml --- +--- vector.yaml --- test 'check_simple_log': @@ -137,28 +141,23 @@ As we introduce new transforms to our config we can change the test output to check the latest transform. Or, occasionally, we can add conditions to an output in order to turn it into a regression test: -```toml title="vector.toml" -[[tests]] - name = "check_simple_log" - - [[tests.inputs]] - insert_at = "foo" - type = "raw" - value = "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" - - # This is now a regression test - [[tests.outputs]] - extract_from = "foo" - [[tests.outputs.conditions]] - type = "vrl" - source = """ - assert_eq!(.message, "Sorry, I'm busy this week Cecil") - """ - - # And we add a new output without conditions for inspecting - # a new transform - [[tests.outputs]] - extract_from = "bar" +```yaml title="vector.yaml" +tests: + - name: "check_simple_log" + inputs: + - insert_at: "foo" + type: "raw" + value: "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" + outputs: + # This is now a regression test + - extract_from: "foo" + conditions: + - type: "vrl" + source: | + assert_eq!(.message, "Sorry, I'm busy this week Cecil") + # And we add a new output without conditions for inspecting + # a new transform + - extract_from: "bar" ``` How many tests you add is at your discretion, but you probably don't need to @@ -175,10 +174,10 @@ With Vector you can split a config down into as many files as you like and run them all as a larger topology: ```bash -# These three examples run the same two configs together: -vector -c ./configs/foo.toml -c ./configs/bar.toml -vector -c ./configs/*.toml -vector -c ./configs/foo.toml ./configs/bar.toml +# These three examples run the same two configs together: +vector -c ./configs/foo.yaml -c ./configs/bar.yaml +vector -c ./configs/*.yaml +vector -c ./configs/foo.yaml ./configs/bar.yaml ``` If you have a large chain of components it's a good idea to break them out into @@ -193,52 +192,58 @@ With Vector you can define a component configuration inside a component type fol Let's take an example with the following configuration file: -```toml title="vector.toml" -[sources.syslog] -type = "syslog" -address = "0.0.0.0:514" -max_length = 42000 -mode = "tcp" - -[transforms.change_fields] -type = "remap" -inputs = ["syslog"] -source = """ -.new_field = "some value" -""" - -[sinks.stdout] -type = "console" -inputs = ["change_fields"] -target = "stdout" -encoding.codec = "json" +```yaml title="vector.yaml" +sources: + syslog: + type: "syslog" + address: "0.0.0.0:514" + max_length: 42000 + mode: "tcp" + +transforms: + change_fields: + type: "remap" + inputs: + - "syslog" + source: | + .new_field = "some value" + +sinks: + stdout: + type: "console" + inputs: + - "change_fields" + target: "stdout" + encoding: + codec: "json" ``` -We can extract the `syslog` source in the file `/etc/vector/sources/syslog.toml` +We can extract the `syslog` source in the file `/etc/vector/sources/syslog.yaml` -```toml title="syslog.toml" -type = "syslog" -address = "0.0.0.0:514" -max_length = 42000 -mode = "tcp" +```yaml title="syslog.yaml" +type: "syslog" +address: "0.0.0.0:514" +max_length: 42000 +mode: "tcp" ``` -The `change_fields` transform in the file `/etc/vector/transforms/change_fields.toml` +The `change_fields` transform in the file `/etc/vector/transforms/change_fields.yaml` -```toml title="change_fields.toml" -type = "remap" -inputs = ["syslog"] -source = """ -.new_field = "some value" -""" +```yaml title="change_fields.yaml" +type: "remap" +inputs: + - "syslog" +source: | + .new_field = "some value" ``` -And the `stdout` sink in the file `/etc/vector/sinks/stdout.toml` +And the `stdout` sink in the file `/etc/vector/sinks/stdout.yaml` -```toml title="stdout.toml" -type = "console" -inputs = ["change_fields"] -target = "stdout" +```yaml title="stdout.yaml" +type: "console" +inputs: + - "change_fields" +target: "stdout" ``` And for Vector to look for the configuration in the component type related folders, diff --git a/website/content/en/guides/level-up/managing-schemas.md b/website/content/en/guides/level-up/managing-schemas.md index ab48bee63f6e8..168e330c8312d 100644 --- a/website/content/en/guides/level-up/managing-schemas.md +++ b/website/content/en/guides/level-up/managing-schemas.md @@ -41,13 +41,13 @@ By default, Vector primarily operates on three fields: `host`, `message`, and `t Vector sets these fields on logs as it ingests data (from a [source][docs.sources]). It may be that your data does not follow this convention. In this case you can modify the global defaults for all incoming data in the `log_schema` -section of your `vector.toml`. +section of your `vector.yaml`. -```toml title="vector.toml" -[log_schema] -host_key = "instance" # default "host" -message_key = "info" # default "message" -timestamp_key = "datetime" # default "timestamp" +```yaml title="vector.yaml" +log_schema: + host_key: "instance" # default "host" + message_key: "info" # default "message" + timestamp_key: "datetime" # default "timestamp" # Sources, transforms, and sinks... ``` @@ -65,15 +65,16 @@ and you'll likely need to configure this at a more fine-grained level. Some services will produce logs with the timestamp field mapped to `@timestamp` or some other value. If your vector pipeline is only working with data passing through these systems, you can add the following to your -`vector.toml`: +`vector.yaml`: -```toml title="vector.toml" -[log_schema] - timestamp_key = "@timestamp" # Applies to all sources, sinks, and transforms! +```yaml title="vector.yaml" +log_schema: + timestamp_key: "@timestamp" # Applies to all sources, sinks, and transforms! -[sources.my_naming_confused_source] - type = "logplex" - address = "0.0.0.0:8088" +sources: + my_naming_confused_source: + type: "logplex" + address: "0.0.0.0:8088" ``` @@ -91,24 +92,24 @@ Commonly you'll want to do this near either the source or sink of your pipeline. A transform of this type looks like this: -```toml title="vector.toml" -[transforms.strip_personal_details] -type = "remap" -inputs = ["my-source-id"] -source = ''' - del(.email, .passport_number) -''' +```yaml title="vector.yaml" +transforms: + strip_personal_details: + type: "remap" + inputs: ["my-source-id"] + source: | + del(.email, .passport_number) ``` The `remap` transform has a wealth of mapping functions, and in cases where we wish to flip this concept and drop all fields except for a list of exceptions we can do that with the `only_fields` function: -```toml title="vector.toml" -[transforms.strip_personal_details] -type = "remap" -inputs = ["my-source-id"] -source = ''' - only_fields(.timestamp, .message, .host, .user_id) -''' +```yaml title="vector.yaml" +transforms: + strip_personal_details: + type: "remap" + inputs: ["my-source-id"] + source: | + only_fields(.timestamp, .message, .host, .user_id) ``` ### Example: Filtering data for GDPR compliance @@ -127,51 +128,54 @@ config, just a little example!) We can build a config that will do the first part of this, but we'll just output to console for ease of this example. -```toml title="vector.toml" -data_dir = "./data" -dns_servers = [] - -[sources.application] -max_length = 102400 -type = "stdin" - -[transforms.parse] -inputs = ["application"] -type = "remap" -source = ''' -. = parse_json!(.message) -''' - -[transforms.not_gdpr] -type = "filter" -inputs = ["parse"] -condition = ".gdpr == false" - -[transforms.gdpr_to_strip] -type = "filter" -inputs = ["parse"] -condition = ".gdpr == true" - -[transforms.gdpr_stripped] -type = "remap" -inputs = ["gdpr_to_strip"] -source = "del(.email)" - -[sinks.console] -healthcheck = true -inputs = ["not_gdpr", "gdpr_stripped"] -type = "console" -encoding.codec = "json" -[sinks.console.buffer] -type = "memory" -max_events = 500 -when_full = "block" +```yaml title="vector.yaml" +data_dir: "./data" +dns_servers: [] + +sources: + application: + max_length: 102400 + type: "stdin" + +transforms: + parse: + inputs: ["application"] + type: "remap" + source: | + . = parse_json!(.message) + + not_gdpr: + type: "filter" + inputs: ["parse"] + condition: ".gdpr == false" + + gdpr_to_strip: + type: "filter" + inputs: ["parse"] + condition: ".gdpr == true" + + gdpr_stripped: + type: "remap" + inputs: ["gdpr_to_strip"] + source: "del(.email)" + +sinks: + console: + healthcheck: true + inputs: ["not_gdpr", "gdpr_stripped"] + type: "console" + encoding: + codec: "json" + buffer: + type: "memory" + max_events: 500 + when_full: "block" ``` Let's have a look: ```bash -$ cat <<-EOF | cargo run -- --config test.toml +$ cat <<-EOF | cargo run -- --config test.yaml { "id": "user1", "gdpr": false, "email": "us-user1@datadoghq.com" } { "id": "user2", "gdpr": false, "email": "us-user2@datadoghq.com" } { "id": "user3", "gdpr": true, "email": "eu-user3@datadoghq.com" } @@ -202,15 +206,17 @@ The applications for this include some of the reasons discussed in Lets take a look at what that might look like: -```toml title="vector.toml" -[sinks.output] - inputs = ["demo_logs"] - type = "kafka" - - # Put events in the host specific topic. - topic = "{{service}}" - encoding.except_fields = ["service"] # Remove this field now and save some bytes - # ... +```yaml title="vector.yaml" +sinks: + output: + inputs: ["demo_logs"] + type: "kafka" + + # Put events in the host specific topic. + topic: "{{service}}" + encoding: + except_fields: ["service"] # Remove this field now and save some bytes + # ... ``` {{< warning >}} @@ -222,14 +228,14 @@ a field which you want templatable open an issue and let us know. It's fairly common for one part of your pipeline to expect a field to be named differently than another part! The `remap` transform can also be used to slide data around for you. -```toml title="vector.toml" -[transforms.rename_timestamp] - type = "remap" - inputs = ["source0"] - source = ''' - ."@timestamp" = .timestamp - del(.timestamp) - ''' +```yaml title="vector.yaml" +transforms: + rename_timestamp: + type: "remap" + inputs: ["source0"] + source: | + ."@timestamp" = .timestamp + del(.timestamp) ``` Other times you might need to concatenate fields together, or perform arithmetic on their numerical values, the [`remap`][docs.transforms.remap] transform can be used to do all of these things. @@ -246,14 +252,14 @@ It's useful for when: Let's pretend one of your teammates falsely assumed folks always have first and last names, so we have a `first_name` and a `last_name` field coming from a source, and we'd like to output a `name` field to a sink. -```toml title="vector.toml" -[transforms.moosh_names] - type = "remap" - inputs = ["source0"] - source = ''' - .name = .first_name + " " + .last_name - del(.first_name, .last_name) - ''' +```yaml title="vector.yaml" +transforms: + moosh_names: + type: "remap" + inputs: ["source0"] + source: | + .name = .first_name + " " + .last_name + del(.first_name, .last_name) ``` ## Coercing Data Types @@ -263,14 +269,14 @@ should be a number, or vice versa. Gadzooks! The [`remap`][docs.transforms.remap] transform is also the correct tool for this job! -```toml title="vector.toml" -[transforms.correct_source_types] - type = "remap" - inputs = ["source0"] - source = ''' - .count = int(.count) - .date = timestamp(.date, "%F") - ''' +```yaml title="vector.yaml" +transforms: + correct_source_types: + type: "remap" + inputs: ["source0"] + source: | + .count = int(.count) + .date = timestamp(.date, "%F") ``` Remember that you can follow the coercion mappings with `del` or `only_fields` functions, empowering it to drop @@ -286,12 +292,12 @@ To do this we'll use the `timestamp` function with a format string argument. To [`strftime`](https://docs.rs/chrono/0.4.10/chrono/format/strftime/index.html) documentation. Let's ship some Canadian friendly logs up to the great white north! -```toml title="vector.toml" -[transforms.format_timestamp] - type = "remap" - source = ''' - .timestamp = timestamp(.timestamp, "%Y/%m/%d:%H:%M:%S %z") - ''' +```yaml title="vector.yaml" +transforms: + format_timestamp: + type: "remap" + source: | + .timestamp = timestamp(.timestamp, "%Y/%m/%d:%H:%M:%S %z") ``` ## Working with data formats @@ -305,12 +311,14 @@ supported. In these cases, you can use the `encoding` option. The [`console`][docs.sinks.console] sink supports both `json` and `text` as its output format -```toml title="vector.toml" -[sinks.print] - type = "console" - inputs = ["source0"] - target = "stdout" - encoding.codec = "json" +```yaml title="vector.yaml" +sinks: + print: + type: "console" + inputs: ["source0"] + target: "stdout" + encoding: + codec: "json" ``` You can also use a transform like [`remap`][docs.transforms.remap] or to parse out diff --git a/website/content/en/guides/level-up/vector-tap-guide.md b/website/content/en/guides/level-up/vector-tap-guide.md index c0622994311fa..0ace743aff5d3 100644 --- a/website/content/en/guides/level-up/vector-tap-guide.md +++ b/website/content/en/guides/level-up/vector-tap-guide.md @@ -31,27 +31,26 @@ To start, we'll reference the following base configuration, but feel free to substitute your own. Just note that the [Vector API] must be enabled for `vector tap` to work. See [under the hood](#under-the-hood) for more details. -```toml -[api] -enabled = true - -[sources.in] -type = "demo_logs" -format = "shuffle" -lines = [ - "test1", - "test2", -] - -[sinks.out] -type = "blackhole" -inputs = ["in*"] +```yaml +api: + enabled: true + +sources: + in: + type: "demo_logs" + format: "shuffle" + lines: ["test1", "test2"] + +sinks: + out: + type: "blackhole" + inputs: ["in*"] ``` Run Vector with this configuration and watch the configuration for changes. ```console -vector --config path/to/config.toml -w +vector --config path/to/config.yaml -w ``` Now run `vector tap`! You should start seeing a stream of @@ -125,14 +124,12 @@ accordingly by re-matching your provided patterns. With `vector tap` running, add the following `demo_logs` source to the base configuration. -```toml -[sources.in-2] -type = "demo_logs" -format = "shuffle" -lines = [ - "new test1", - "new test2", -] +```yaml +sources: + in-2: + type: "demo_logs" + format: "shuffle" + lines: ["new test1", "new test2"] ``` You'll now see events from component `in-2` appear in `tap` output. @@ -155,43 +152,45 @@ troubleshoot a pipeline. We'll use the following Vector configuration. -```toml -[api] -enabled = true - -[sources.in] -type = "demo_logs" -format = "shuffle" -lines = [ - '{ "type": "icecream", "flavor": "strawberry" }', - '{ "type": "icecream", "flavor": "chocolate" }', - '{ "type": "icecream", "flavor": "wasabi" }', -] - -[transforms.picky] -type = "remap" -inputs = ["in"] -drop_on_abort = true -reroute_dropped = true -source = ''' - if .flavor == "strawberry" { - .happiness = 10 - } else if .flavor == "chocolate" { - .happiness = 5 - } else { - abort - } -''' - -[sinks.store] -type = "console" -inputs = ["picky"] -target = "stdout" -encoding.codec = "json" - -[sinks.trash] -type = "blackhole" -inputs = ["picky.dropped"] +```yaml +api: + enabled: true + +sources: + in: + type: "demo_logs" + format: "shuffle" + lines: + - '{ "type": "icecream", "flavor": "strawberry" }' + - '{ "type": "icecream", "flavor": "chocolate" }' + - '{ "type": "icecream", "flavor": "wasabi" }' + +transforms: + picky: + type: "remap" + inputs: ["in"] + drop_on_abort: true + reroute_dropped: true + source: | + if .flavor == "strawberry" { + .happiness = 10 + } else if .flavor == "chocolate" { + .happiness = 5 + } else { + abort + } + +sinks: + store: + type: "console" + inputs: ["picky"] + target: "stdout" + encoding: + codec: "json" + + trash: + type: "blackhole" + inputs: ["picky.dropped"] ``` Running this configuration, we expect to see our favorite ice cream logs appear diff --git a/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md b/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md index 141437904cc99..90799c6d1ef9d 100644 --- a/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md +++ b/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md @@ -26,40 +26,39 @@ mission-critical production pipelines that are collaborated on. Let's look at a basic example that uses the [`regex_parser` transform][docs.transforms.regex_parser] to parse log lines: -```toml title="vector.toml" -[sources.my_logs] - type = "file" - include = ["/var/log/my-app.log"] - -[transforms.parser] - inputs = ["my_logs"] - type = "regex_parser" - regex = "^(?P[\\w\\-:\\+]+) (?P\\w+) (?P.*)$" - -[[tests]] - name = "verify_regex" - - [tests.input] - insert_at = "parser" - type = "raw" - value = "2019-11-28T12:00:00+00:00 info Hello world" - - [[tests.outputs]] - extract_from = "parser" - - [[tests.outputs.conditions]] - type = "check_fields" - "timestamp.equals" = "2019-11-28T12:00:00+00:00" - "level.equals" = "info" - "message.equals" = "Hello world" +```yaml title="vector.yaml" +sources: + my_logs: + type: "file" + include: ["/var/log/my-app.log"] + +transforms: + parser: + inputs: ["my_logs"] + type: "regex_parser" + regex: "^(?P[\\w\\-:\\+]+) (?P\\w+) (?P.*)$" + +tests: + - name: "verify_regex" + input: + insert_at: "parser" + type: "raw" + value: "2019-11-28T12:00:00+00:00 info Hello world" + outputs: + - extract_from: "parser" + conditions: + - type: "check_fields" + "timestamp.equals": "2019-11-28T12:00:00+00:00" + "level.equals": "info" + "message.equals": "Hello world" ``` And you can run the tests via the new `test` subcommand: ```sh -$ vector test ./vector.toml -Running ./vector.toml tests -Test ./vector.toml: verify_regex ... passed +$ vector test ./vector.yaml +Running ./vector.yaml tests +Test ./vector.yaml: verify_regex ... passed ``` ## Why? diff --git a/website/content/en/highlights/2019-12-13-custom-dns.md b/website/content/en/highlights/2019-12-13-custom-dns.md index 3fcbd88fd24c0..a9d0088aa4675 100644 --- a/website/content/en/highlights/2019-12-13-custom-dns.md +++ b/website/content/en/highlights/2019-12-13-custom-dns.md @@ -20,8 +20,8 @@ servers in your configs. The configuration isn't complicated, it's a global array field `dns_servers`: -```toml -dns_servers = ["0.0.0.0:53"] +```yaml +dns_servers: ["0.0.0.0:53"] ``` When `dns_servers` is set Vector will ignore the system configuration and use diff --git a/website/content/en/highlights/2019-12-16-ec2-metadata.md b/website/content/en/highlights/2019-12-16-ec2-metadata.md index 58d92e577ad63..117441d3db43d 100644 --- a/website/content/en/highlights/2019-12-16-ec2-metadata.md +++ b/website/content/en/highlights/2019-12-16-ec2-metadata.md @@ -21,19 +21,19 @@ our brand spanking new [`aws_ec2_metadata` transform][docs.transforms.aws_ec2_me Configuration isn't complicated, just add and hook up the transform. If you don't want all enrichments added then white-list them with the `fields` option: -```toml -[transforms.fill_me_up] - type = "aws_ec2_metadata" - inputs = ["my-source-id"] - fields = [ - "instance-id", - "local-hostname", - "public-hostname", - "public-ipv4", - "ami-id", - "availability-zone", - "region", - ] +```yaml +transforms: + fill_me_up: + type: "aws_ec2_metadata" + inputs: ["my-source-id"] + fields: + - "instance-id" + - "local-hostname" + - "public-hostname" + - "public-ipv4" + - "ami-id" + - "availability-zone" + - "region" ``` For more guidance get on the [reference page][docs.transforms.aws_ec2_metadata]. diff --git a/website/content/en/highlights/2020-01-07-prometheus-source.md b/website/content/en/highlights/2020-01-07-prometheus-source.md index 2d1797aff9749..45ca46ead7bc3 100644 --- a/website/content/en/highlights/2020-01-07-prometheus-source.md +++ b/website/content/en/highlights/2020-01-07-prometheus-source.md @@ -25,11 +25,12 @@ metrics data model and tested our interoperability between metrics sources. To use it simply add the source config and point it towards the hosts you wish to scrape: -```toml -[sources.my_source_id] - type = "prometheus_scrape" - hosts = ["http://localhost:9090"] - scrape_interval_secs = 1 +```yaml +sources: + my_source_id: + type: "prometheus_scrape" + hosts: ["http://localhost:9090"] + scrape_interval_secs: 1 ``` For more guidance get on the [reference page][docs.sources.prometheus]. diff --git a/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md b/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md index e8d6f6a018f61..d1aadc19bc248 100644 --- a/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md +++ b/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md @@ -21,9 +21,10 @@ will _not_ index any fields by default. In order to mark desired fields as indexed you can use the optional configuration option `indexed_fields`: -```toml title="vector.toml" - [sinks.my_sink_id] - type = "splunk_hec" - inputs = ["my-source-id"] -+ indexed_fields = ["foo", "bar"] +```yaml title="vector.yaml" + sinks: + my_sink_id: + type: "splunk_hec" + inputs: ["my-source-id"] ++ indexed_fields: ["foo", "bar"] ``` diff --git a/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md b/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md index 2dc7a049aba31..d1af3aaae1249 100644 --- a/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md +++ b/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md @@ -20,10 +20,11 @@ events. This can be a very difficult and frustrating problem to solve with other tools (we speak from experience). In this release, Vector solves this automatically with a new `auto_partial_merge` option in the `docker_logs` source. -```toml title="vector.toml" -[sources.my_source_id] - type = "docker_logs" - auto_partial_merge = true +```yaml title="vector.yaml" +sources: + my_source_id: + type: "docker_logs" + auto_partial_merge: true ``` We love assimilation and look forward to a future where our individualistic diff --git a/website/content/en/highlights/2020-02-14-global-log-schema.md b/website/content/en/highlights/2020-02-14-global-log-schema.md index 6949d0d409fae..213030fd5a3d5 100644 --- a/website/content/en/highlights/2020-02-14-global-log-schema.md +++ b/website/content/en/highlights/2020-02-14-global-log-schema.md @@ -24,11 +24,11 @@ the default names for the [`message_key`][docs.global-options#message_key], [`host_key`][docs.global-options#host_key], [`timestamp_key`][docs.global-options#host_key], and more: -```toml title="vector.toml" -[log_schema] - host_key = "host" # default - message_key = "message" # default - timestamp_key = "timestamp" # default +```yaml title="vector.yaml" +log_schema: + host_key: "host" # default + message_key: "message" # default + timestamp_key: "timestamp" # default ``` Why is this useful? diff --git a/website/content/en/highlights/2020-02-21-file-source-multiline-support.md b/website/content/en/highlights/2020-02-21-file-source-multiline-support.md index 762faf8c73e4f..f09c9f4789c95 100644 --- a/website/content/en/highlights/2020-02-21-file-source-multiline-support.md +++ b/website/content/en/highlights/2020-02-21-file-source-multiline-support.md @@ -34,16 +34,16 @@ foobar.rb:6:in `/': divided by 0 (ZeroDivisionError) You can merge them with the following config: -```toml title="vector.toml" -[sources.my_file_source] - type = "file" - # ... - - [sources.my_file_source.multiline] - start_pattern = "^[^\\s]" - mode = "continue_through" - condition_pattern = "^[\\s]+from" - timeout_ms = 1000 +```yaml title="vector.yaml" +sources: + my_file_source: + type: "file" + # ... + multiline: + start_pattern: "^[^\\s]" + mode: "continue_through" + condition_pattern: "^[\\s]+from" + timeout_ms: 1000 ``` And if this doesn't do it, you can always fall back to the [`lua` transform][docs.transforms.lua]. diff --git a/website/content/en/highlights/2020-02-24-swimlanes-transform.md b/website/content/en/highlights/2020-02-24-swimlanes-transform.md index 815694d51fea5..29e7c67cc280a 100644 --- a/website/content/en/highlights/2020-02-24-swimlanes-transform.md +++ b/website/content/en/highlights/2020-02-24-swimlanes-transform.md @@ -16,15 +16,16 @@ The new [`swimlanes` transform][docs.transforms.swimlanes] makes it much easier to configure conditional branches of transforms and sinks. For example, you can easily create [if/else pipelines][docs.transforms.swimlanes#examples]. -```toml title="vector.toml" -[transforms.lanes] - types = "swimlanes" - - [transforms.my_transform_id.lanes.errors] - "level.eq" = "error" - - [transforms.my_transform_id.lanes.not_errors] - "level.neq" = "error" +```yaml title="vector.yaml" +transforms: + lanes: + types: "swimlanes" + my_transform_id: + lanes: + errors: + "level.eq": "error" + not_errors: + "level.neq": "error" ``` Remember to occasionally let your branches mingle so that they don't completely diff --git a/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md b/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md index 08032a186b521..7f6e5833a431d 100644 --- a/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md +++ b/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md @@ -23,11 +23,13 @@ Vector has deprecated the root-level `encoding` option in favor of new Upgrading is easy: -```toml title="vector.toml" - [sinks.my-sink] - type = "..." -- encoding = "json" -+ encoding.codec = "json" -+ encoding.except_fields = ["_meta"] # optional -+ encoding.timestamp_format = "rfc3339" # optional +```yaml title="vector.yaml" + sinks: + my-sink: + type: "..." +- encoding: "json" ++ encoding: ++ codec: "json" ++ except_fields: ["_meta"] # optional ++ timestamp_format: "rfc3339" # optional ``` diff --git a/website/content/en/highlights/2020-03-10-dedupe-transform.md b/website/content/en/highlights/2020-03-10-dedupe-transform.md index 4a596b98f448b..44b68d9d489d1 100644 --- a/website/content/en/highlights/2020-03-10-dedupe-transform.md +++ b/website/content/en/highlights/2020-03-10-dedupe-transform.md @@ -22,14 +22,16 @@ mistakes that accidentally duplicate logs. This mistake can easily double Simply add the transform to your pipeline: -```toml -[transforms.my_transform_id] - # General - type = "dedupe" # required - inputs = ["my-source-id"] # required +```yaml +transforms: + my_transform_id: + # General + type: "dedupe" # required + inputs: ["my-source-id"] # required - # Fields - fields.match = ["timestamp", "host", "message"] # optional, default + # Fields + fields: + match: ["timestamp", "host", "message"] # optional, default ``` {{< success >}} diff --git a/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md b/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md index c6a053786906a..b184613dbbcfb 100644 --- a/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md +++ b/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md @@ -20,13 +20,14 @@ protect against this we built a new Getting started is easy. Simply add this component to your pipeline: -```toml title="vector.toml" -[transforms.tag_protection] - type = "tag_cardinality_limit" - inputs = ["my-source-id"] - limit_exceeded_action = "drop_tag" - mode = "exact" - value_limit = 500 +```yaml title="vector.yaml" +transforms: + tag_protection: + type: "tag_cardinality_limit" + inputs: ["my-source-id"] + limit_exceeded_action: "drop_tag" + mode: "exact" + value_limit: 500 ``` {{< success >}} diff --git a/website/content/en/highlights/2020-03-31-filter-transform.md b/website/content/en/highlights/2020-03-31-filter-transform.md index b1524f5d60910..5409db5a6b28a 100644 --- a/website/content/en/highlights/2020-03-31-filter-transform.md +++ b/website/content/en/highlights/2020-03-31-filter-transform.md @@ -20,15 +20,17 @@ transform since it is much more expressive. ## Get Started -```toml title="vector.toml" -[transforms.haproxy_errors] - # General - type = "filter" - inputs = ["my-source-id"] +```yaml title="vector.yaml" +transforms: + haproxy_errors: + # General + type: "filter" + inputs: ["my-source-id"] - # Conditions - condition."level.eq" = "error" - condition."service.eq" = "haproxy" + # Conditions + condition: + "level.eq": "error" + "service.eq": "haproxy" ``` Check out the [docs][docs.transforms.filter] for a fill list of available diff --git a/website/content/en/highlights/2020-04-01-more-condition-predicates.md b/website/content/en/highlights/2020-04-01-more-condition-predicates.md index 277e780274890..68d56c1b9d38e 100644 --- a/website/content/en/highlights/2020-04-01-more-condition-predicates.md +++ b/website/content/en/highlights/2020-04-01-more-condition-predicates.md @@ -31,10 +31,12 @@ following predicates were added: For example, you can filter all messages that contain the `error` term with the new `contains` predicate: -```toml -[transforms.errors] - type = "filter" - condition."message.contain" = "error" +```yaml +transforms: + errors: + type: "filter" + condition: + "message.contain": "error" ``` The world is your oyster. diff --git a/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md b/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md index dc5a36becf1a4..6124b4d055aa7 100644 --- a/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md +++ b/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md @@ -15,15 +15,17 @@ Working with `openssl` isn't very fun, and we don't want to inflict that on you. This is particularly useful if you have a socket source: -```toml title="vector.toml" -[sources.tls] - type = "socket" - address = "0.0.0.0:6514" - mode = "tcp" - tls.enabled = true - tls.crt_path = "cert.pfx" - tls.ca_path = "ca.pem" # Now supported: More complicated PEMS! - tls.verify_certificate = true +```yaml title="vector.yaml" +sources: + tls: + type: "socket" + address: "0.0.0.0:6514" + mode: "tcp" + tls: + enabled: true + crt_path: "cert.pfx" + ca_path: "ca.pem" # Now supported: More complicated PEMS! + verify_certificate: true ``` If it doesn't, that's a bug. [**Report it.**][urls.new_bug_report] We squash bugs. diff --git a/website/content/en/highlights/2020-07-10-add-reduce-transform.md b/website/content/en/highlights/2020-07-10-add-reduce-transform.md index 96b757c11f88c..fc7034c4cd871 100644 --- a/website/content/en/highlights/2020-07-10-add-reduce-transform.md +++ b/website/content/en/highlights/2020-07-10-add-reduce-transform.md @@ -54,42 +54,49 @@ Then output this (but not formatted so nicely!): We'll run this config: -```toml title=vector.toml -data_dir = "tmp" - -[sources.source0] - include = ["input.log"] - start_at_beginning = true - type = "file" - fingerprinting.strategy = "device_and_inode" - -[transforms.transform0] - inputs = ["source0"] - type = "json_parser" - field = "message" - -[transforms.transform1] - inputs = ["transform0"] - type = "reduce" - identifier_fields = ["request_id"] - ends_when.type = "check_fields" - ends_when."response_status.exists" = true - merge_strategies.message = "discard" - merge_strategies.query = "discard" - merge_strategies.template = "discard" - merge_strategies.query_duration_ms = "sum" - merge_strategies.render_duration_ms = "sum" - merge_strategies.response_duration_ms = "sum" - -[sinks.sink0] - healthcheck = true - inputs = ["transform1"] - type = "file" - path = "output.log" - encoding = "ndjson" - buffer.type = "memory" - buffer.max_events = 500 - buffer.when_full = "block"s +```yaml title=vector.yaml +data_dir: "tmp" + +sources: + source0: + include: ["input.log"] + start_at_beginning: true + type: "file" + fingerprinting: + strategy: "device_and_inode" + +transforms: + transform0: + inputs: ["source0"] + type: "json_parser" + field: "message" + + transform1: + inputs: ["transform0"] + type: "reduce" + identifier_fields: ["request_id"] + ends_when: + type: "check_fields" + "response_status.exists": true + merge_strategies: + message: "discard" + query: "discard" + template: "discard" + query_duration_ms: "sum" + render_duration_ms: "sum" + response_duration_ms: "sum" + +sinks: + sink0: + healthcheck: true + inputs: ["transform1"] + type: "file" + path: "output.log" + encoding: "ndjson" + buffer: + type: "memory" + max_events: 500 + when_full: "block" ``` We hope you find this useful! diff --git a/website/content/en/highlights/2020-09-18-adaptive-concurrency.md b/website/content/en/highlights/2020-09-18-adaptive-concurrency.md index a7236e736224d..925f66081b7b6 100644 --- a/website/content/en/highlights/2020-09-18-adaptive-concurrency.md +++ b/website/content/en/highlights/2020-09-18-adaptive-concurrency.md @@ -25,11 +25,13 @@ inspired by TCP congestion control algorithms. This feature, like all Vector features, will begin its life in public beta and be available on an opt-in basis. To get it, enable it for each sink: -```toml -[sinks.my-sink] -type = "..." # any http-based sink -request.concurrency = "adaptive" -# and remove the request.rate_limit_* settings +```yaml +sinks: + my-sink: + type: "..." # any http-based sink + request: + concurrency: "adaptive" + # and remove the request.rate_limit_* settings ``` [announcement]: /blog/adaptive-request-concurrency/ diff --git a/website/content/en/highlights/2020-10-27-metrics-integrations.md b/website/content/en/highlights/2020-10-27-metrics-integrations.md index 50bd6e8132ca1..6666f50fd247b 100644 --- a/website/content/en/highlights/2020-10-27-metrics-integrations.md +++ b/website/content/en/highlights/2020-10-27-metrics-integrations.md @@ -39,14 +39,16 @@ the single agent for all of your logs, metrics, and traces. To get started with these sources, define them and go: -```toml -[sources.host_metrics] -type = "host_metrics" # or apache_metrics, mongodb_metrics, or internal_metrics +```yaml +sources: + host_metrics: + type: "host_metrics" # or apache_metrics, mongodb_metrics, or internal_metrics # Then connect them to a sink: -[sinks.prometheus] -type = "prometheus" -inputs = ["host_metrics"] +sinks: + prometheus: + type: "prometheus" + inputs: ["host_metrics"] ``` Tada! One agent for all of your data. Check out the [docs][docs] for more diff --git a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md index 604b85bfc4bf7..965b42c6b0365 100644 --- a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md +++ b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md @@ -31,17 +31,20 @@ Prometheus operators can route a stream of Prometheus data to the archiving solution of their choice. For this use case we recommend object stores for their cheap and durable qualities: -```toml title="vector.toml" -[sources.prometheus] - type = "prometheus_remote_write" - -[transforms.convert] - type = "metric_to_log" - inputs = ["prometheus"] - -[sinks.backup] - type = "aws_s3" - inputs = ["convert"] +```yaml title="vector.yaml" +sources: + prometheus: + type: "prometheus_remote_write" + +transforms: + convert: + type: "metric_to_log" + inputs: ["prometheus"] + +sinks: + backup: + type: "aws_s3" + inputs: ["convert"] ``` Swap `aws_s3` with `gcp_cloud_storage` or other object stores. @@ -68,13 +71,15 @@ To get started, setup the new your metrics to [Datadog][datadog], [New Relic][new_relic], [Influx][influx], [Elasticsearch][elastic], and [many other sinks][sinks]: -```toml title="vector.toml" -[sources.prometheus] - type = "prometheus_remote_write" +```yaml title="vector.yaml" +sources: + prometheus: + type: "prometheus_remote_write" -[sinks.datadog] - type = "datadog_metrics" - inputs = ["prometheus"] +sinks: + datadog: + type: "datadog_metrics" + inputs: ["prometheus"] ``` [chronosphere]: https://chronosphere.io/ diff --git a/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md b/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md index db44d80923daf..f977ab64a4de8 100644 --- a/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md +++ b/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md @@ -12,9 +12,9 @@ badges: --- To ensure Vector fits into existing workflows, like Kubernetes, we've added -support for JSON and YAML config formats in addition to TOML. TOML will -continue to be our default format, but users are welcome to use JSON and YAML -as they see fit. +support for JSON and YAML config formats in addition to TOML. YAML is now the +recommended and default configuration format, but users are welcome to use +TOML and JSON as they see fit. ## Use cases diff --git a/website/content/en/highlights/2020-12-23-graphql-api.md b/website/content/en/highlights/2020-12-23-graphql-api.md index 70c40ebfac28a..e749377c8dd05 100644 --- a/website/content/en/highlights/2020-12-23-graphql-api.md +++ b/website/content/en/highlights/2020-12-23-graphql-api.md @@ -28,10 +28,10 @@ The GraphQL API for Vector is **disabled by default**. We want to keep Vector's behavior as predictable and secure as possible, so we chose to make the feature opt-in. To enable the API, add this to your Vector config: -```toml -[api] -enabled = true -address = "127.0.0.1:8686" # optional. Change IP/port if required +```yaml +api: + enabled: true + address: "127.0.0.1:8686" # optional. Change IP/port if required ``` ## Read more diff --git a/website/content/en/highlights/2020-12-23-internal-logs-source.md b/website/content/en/highlights/2020-12-23-internal-logs-source.md index b90f1899975e8..fb8ae9aed167e 100644 --- a/website/content/en/highlights/2020-12-23-internal-logs-source.md +++ b/website/content/en/highlights/2020-12-23-internal-logs-source.md @@ -38,24 +38,27 @@ metrics and modify and ship them however you wish. Here's an example Vector configuration that ships Vector's logs to Splunk and allows its internal metrics to be scraped by [Prometheus]: -```toml -[sources.vector_logs] -type = "internal_logs" - -[sources.vector_metrics] -type = "internal_metrics" - -[sinks.splunk] -type = "splunk_hec" -inputs = ["vector_logs"] -endpoint = "https://my-account.splunkcloud.com" -token = "${SPLUNK_HEC_TOKEN}" -encoding.codec = "json" - -[sinks.prometheus] -type = "prometheus" -inputs = ["vector_metrics"] -address = "0.0.0.0:9090" +```yaml +sources: + vector_logs: + type: "internal_logs" + + vector_metrics: + type: "internal_metrics" + +sinks: + splunk: + type: "splunk_hec" + inputs: ["vector_logs"] + endpoint: "https://my-account.splunkcloud.com" + token: "${SPLUNK_HEC_TOKEN}" + encoding: + codec: "json" + + prometheus: + type: "prometheus" + inputs: ["vector_metrics"] + address: "0.0.0.0:9090" ``` [internal_logs]: /docs/reference/configuration/sources/internal_logs diff --git a/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md b/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md index 389e85dbf50ff..35e95c6444eba 100644 --- a/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md +++ b/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md @@ -18,14 +18,17 @@ metric events through Kafka. Metrics events are encoded into a format that mimics our [internal metrics data model], ideal for custom consumers on the other end. Getting started is easy: -```toml -[sources.host_metrics] -type = "host_metrics" +```yaml +sources: + host_metrics: + type: "host_metrics" -[sinks.kafka] -type = "kafka" -inputs = ["host_metrics"] -encoding.codec = "json" +sinks: + kafka: + type: "kafka" + inputs: ["host_metrics"] + encoding: + codec: "json" ``` ## Caveats diff --git a/website/content/en/highlights/2021-01-20-wildcard-identifiers.md b/website/content/en/highlights/2021-01-20-wildcard-identifiers.md index 9cff6a92702d1..88f82a0d75539 100644 --- a/website/content/en/highlights/2021-01-20-wildcard-identifiers.md +++ b/website/content/en/highlights/2021-01-20-wildcard-identifiers.md @@ -15,26 +15,28 @@ badges: [PR 6170][pr_6170] introduced wildcards when referencing component names in the `inputs` option. This allows you to build dynamic topologies. This feature comes with one limitation: the wildcard must be at the end of the string. -```toml -[sources.app1_logs] -type = "file" -includes = ["/var/log/app1.log"] +```yaml +sources: + app1_logs: + type: "file" + includes: ["/var/log/app1.log"] -[sources.app2_logs] -type = "file" -includes = ["/var/log/app.log"] + app2_logs: + type: "file" + includes: ["/var/log/app.log"] -[sources.system_logs] -type = "file" -includes = ["/var/log/system.log"] + system_logs: + type: "file" + includes: ["/var/log/system.log"] -[sinks.app_logs] -type = "datadog_logs" -inputs = ["app*"] +sinks: + app_logs: + type: "datadog_logs" + inputs: ["app*"] -[sinks.archive] -type = "aws_s3" -inputs = ["app*", "system_logs"] + archive: + type: "aws_s3" + inputs: ["app*", "system_logs"] ``` [pr_6170]: https://github.com/vectordotdev/vector/pull/6170 diff --git a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md index 9f05bda354d2f..7f5a8f361039d 100644 --- a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md +++ b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md @@ -117,12 +117,12 @@ The following transforms have been deprecated in favor of the new [`remap` trans Deprecation notices have been placed on each of these transforms with example VRL programs that demonstrate how to migrate to the new `remap` transform. For example, migrating from the `json_parser` transform is as simple as: -```toml -[transforms.remap] -type = "remap" -source = ''' -. = merge(., parse_json!(.message)) -''' +```yaml +transforms: + remap: + type: "remap" + source: | + . = merge(., parse_json!(.message)) ``` **You do not need to upgrade immediately. These transforms will not be removed until Vector hits 1.0, a milestone that diff --git a/website/content/en/highlights/2021-02-16-filter-remap-support.md b/website/content/en/highlights/2021-02-16-filter-remap-support.md index 50d47fbabb7d5..26cc72dff7ce5 100644 --- a/website/content/en/highlights/2021-02-16-filter-remap-support.md +++ b/website/content/en/highlights/2021-02-16-filter-remap-support.md @@ -24,20 +24,26 @@ Previously, the `filter` transform required you to specify conditions using The example configuration below shows the same `filter` transform using the old system (`check_fields`) and the new system (`remap`): -```toml -[transforms.filter_out_non_critical] -type = "filter" -inputs = ["http-server-logs"]f - -# Using check_fields -condition.type = "check_fields" -condition.message.status_code.ne = 200 -condition.message.severity.ne = "info" -condition.message.severity.ne = "debug" - -# Using remap -condition.type = "remap" -condition.source = '.status_code != 200 && !includes(["info", "debug"], .severity)' +```yaml +transforms: + filter_out_non_critical: + type: "filter" + inputs: ["http-server-logs"] + + # Using check_fields + condition: + type: "check_fields" + message: + status_code: + ne: 200 + severity: + ne: "info" + # ne: "debug" + + # Using remap + condition: + type: "remap" + source: '.status_code != 200 && !includes(["info", "debug"], .severity)' ``` [filter]: /docs/reference/configuration/transforms/filter diff --git a/website/content/en/highlights/2021-04-21-vector-tap.md b/website/content/en/highlights/2021-04-21-vector-tap.md index be234b5ac9d89..39b89194cbab4 100644 --- a/website/content/en/highlights/2021-04-21-vector-tap.md +++ b/website/content/en/highlights/2021-04-21-vector-tap.md @@ -27,19 +27,22 @@ of this component. For example, given the configuration: -```toml -[api] - enabled = true -[sources.in] - type = "generator" - format = "shuffle" - interval = 1.0 - lines = ["Hello World"] - sequence = true - -[sinks.out] - type = "blackhole" - inputs = ["in"] +```yaml +api: + enabled: true + +sources: + in: + type: "generator" + format: "shuffle" + interval: 1.0 + lines: ["Hello World"] + sequence: true + +sinks: + out: + type: "blackhole" + inputs: ["in"] ``` If you were to run `vector` and then, in another terminal, run `vector tap in`, diff --git a/website/content/en/highlights/2021-04-21-vrl-abort.md b/website/content/en/highlights/2021-04-21-vrl-abort.md index 5e784ab56d505..234fa9a948c37 100644 --- a/website/content/en/highlights/2021-04-21-vrl-abort.md +++ b/website/content/en/highlights/2021-04-21-vrl-abort.md @@ -28,27 +28,32 @@ validation on the event before processing it, discarding any invalid events. Given a config of: -```toml -[sources.in] - type = "generator" - format = "shuffle" - interval = 1.0 - lines = ['{ "message": "valid message", "type": "ok"}', '{ "message": "invalid message", "type": "unknown"}'] +```yaml +sources: + in: + type: "generator" + format: "shuffle" + interval: 1.0 + lines: + - '{ "message": "valid message", "type": "ok"}' + - '{ "message": "invalid message", "type": "unknown"}' -[transforms.remap] - type = "remap" - inputs = ["in"] - source = """ - . |= object!(parse_json!(string!(.message))) - if .type != "ok" { - abort # unknown type - } - """ +transforms: + remap: + type: "remap" + inputs: ["in"] + source: | + . |= object!(parse_json!(string!(.message))) + if .type != "ok" { + abort # unknown type + } -[sinks.out] - type = "console" - inputs = ["remap"] - encoding.codec = "json" +sinks: + out: + type: "console" + inputs: ["remap"] + encoding: + codec: "json" ``` You would expect to see something like: diff --git a/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md b/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md index 38b39874a1b5f..10d03ca5a8f74 100644 --- a/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md +++ b/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md @@ -42,38 +42,42 @@ This means, if you are presently using the deprecated `check_fields` syntax, you For example, if you previously had: -```toml -[transforms.sample] -type = "sample" -inputs = ["in"] -rate = 10 -key_field = "message" -exclude."message.contains" = "error" +```yaml +transforms: + sample: + type: "sample" + inputs: ["in"] + rate: 10 + key_field: "message" + exclude: + "message.contains": "error" ``` You will need to add `exclude.type = "check_fields"` like: -```toml -[transforms.sample] -type = "sample" -inputs = ["in"] -rate = 10 -key_field = "message" -exclude."type" = "check_fields" -exclude."message.contains" = "error" +```yaml +transforms: + sample: + type: "sample" + inputs: ["in"] + rate: 10 + key_field: "message" + exclude: + "type": "check_fields" + "message.contains": "error" ``` To convert this to the new [VRL][VRL] conditions, you would write: -```toml -[transforms.sample] -type = "sample" -inputs = ["in"] -rate = 10 -key_field = "message" -exclude = """ - contains!(.message, "error") -""" +```yaml +transforms: + sample: + type: "sample" + inputs: ["in"] + rate: 10 + key_field: "message" + exclude: | + contains!(.message, "error") ``` We recommend upgrading to the [VRL][VRL] conditions as these are much more powerful than the legacy `check_fields`-style @@ -85,40 +89,42 @@ The `remap` condition type has been renamed `vrl` in this release to better high a [VRL][VRL] program. Most examples of using this condition type have the short-hand condition config of just specifying the [VRL][VRL] program without specifying a `type`. For example: -```toml -[transforms.filter_a] - inputs = ["stdin"] - type = "filter" - condition = ''' +```yaml +transforms: + filter_a: + inputs: ["stdin"] + type: "filter" + condition: | message = if exists(.tags) { .tags.message } else { .message } message == "test filter 1" - ''' ``` Which is automatically a [VRL][VRL] condition. However, if you were specifying the `type` like: -```toml -[transforms.filter_a] -inputs = ["stdin"] -type = "filter" -condition.type = "remap" -condition.source = ''' - message = if exists(.tags) { .tags.message } else { .message } - message == "test filter 1" -''' +```yaml +transforms: + filter_a: + inputs: ["stdin"] + type: "filter" + condition: + type: "remap" + source: | + message = if exists(.tags) { .tags.message } else { .message } + message == "test filter 1" ``` Then you will need to update `type = "remap"` to `type = "vrl"` like: -```toml -[transforms.filter_a] -inputs = ["stdin"] -type = "filter" -condition.type = "vrl" -condition.source = ''' - message = if exists(.tags) { .tags.message } else { .message } - message == "test filter 1" -''' +```yaml +transforms: + filter_a: + inputs: ["stdin"] + type: "filter" + condition: + type: "vrl" + source: | + message = if exists(.tags) { .tags.message } else { .message } + message == "test filter 1" ``` [vrl]: /docs/reference/vrl/ diff --git a/website/content/en/highlights/2021-07-14-vector-graph.md b/website/content/en/highlights/2021-07-14-vector-graph.md index 07a85dbd18f82..b8e7ddcdfc83c 100644 --- a/website/content/en/highlights/2021-07-14-vector-graph.md +++ b/website/content/en/highlights/2021-07-14-vector-graph.md @@ -14,74 +14,78 @@ badges: This release adds a new subcommand `vector graph` to output the topology specified by you Vector configuration as a graph in [DOT][DOT] format. This output can then be visualized using [Graphviz] to produce an image. -For example, if you had a config, `vector.toml`, like: +For example, if you had a config, `vector.yaml`, like: -```toml +```yaml ## ## Sources ## -[sources.internal_metrics] -type = "internal_metrics" +sources: + internal_metrics: + type: "internal_metrics" -[sources.dd_logs] -type = "datadog_logs" # required -acknowledgements = false # optional, default -address = "0.0.0.0:8282" # required + dd_logs: + type: "datadog_logs" # required + acknowledgements: false # optional, default + address: "0.0.0.0:8282" # required -[sources.file_gen] -type = "file" -include = ["/var/log/file_gen/**/*.log"] -read_from = "beginning" + file_gen: + type: "file" + include: ["/var/log/file_gen/**/*.log"] + read_from: "beginning" ## ## Transforms ## -[transforms.remap] -type = "remap" -inputs = ["file_gen"] -source = ''' -.agent_name = "vector" -parsed, err = parse_json(.message) -if err == null { - .message = parsed - .format = "json" -} else { - .format = "ascii" -} -matches = parse_regex!(.file, r'.*/(?P\d+)-(?P\w+).log') -.origin, err = .host + "/" + matches.name + "/" + matches.num -if err != null { - log("Failed to parse origin from file name", level: "error") -} -''' +transforms: + remap: + type: "remap" + inputs: ["file_gen"] + source: | + .agent_name = "vector" + parsed, err = parse_json(.message) + if err == null { + .message = parsed + .format = "json" + } else { + .format = "ascii" + } + matches = parse_regex!(.file, r'.*/(?P\d+)-(?P\w+).log') + .origin, err = .host + "/" + matches.name + "/" + matches.num + if err != null { + log("Failed to parse origin from file name", level: "error") + } ## ## Sinks ## -[sinks.prometheus] -type = "prometheus_exporter" -inputs = ["internal_metrics"] -address = "0.0.0.0:9598" - -[sinks.dd_logs_egress] -type = "datadog_logs" -inputs = ["dd_logs"] -default_api_key = "" -encoding.codec = "json" -request.concurrency = "adaptive" -batch.max_bytes = 5242880 -request.rate_limit_num = 1000 - - -[sinks.blackhole] -type = "blackhole" -inputs = ["remap"] +sinks: + prometheus: + type: "prometheus_exporter" + inputs: ["internal_metrics"] + address: "0.0.0.0:9598" + + dd_logs_egress: + type: "datadog_logs" + inputs: ["dd_logs"] + default_api_key: "" + encoding: + codec: "json" + request: + concurrency: "adaptive" + rate_limit_num: 1000 + batch: + max_bytes: 5242880 + + blackhole: + type: "blackhole" + inputs: ["remap"] ``` -And you ran the new `vector graph --config vector.toml` you would see: +And you ran the new `vector graph --config vector.yaml` you would see: ```dot digraph { @@ -102,7 +106,7 @@ digraph { To render this, if you have [Graphviz] installed, you could do: ```shell -vector graph --config vector.toml | dot -Tpng > graph.png +vector graph --config vector.yaml | dot -Tpng > graph.png ``` To get an image that looks like: diff --git a/website/content/en/highlights/2021-07-16-remap-multiple.md b/website/content/en/highlights/2021-07-16-remap-multiple.md index 5d23a59fabce9..80a0d5fa4d4a6 100644 --- a/website/content/en/highlights/2021-07-16-remap-multiple.md +++ b/website/content/en/highlights/2021-07-16-remap-multiple.md @@ -20,13 +20,13 @@ will create one log event for each event in the array. For example: -```toml -[transforms.remap] -type = "remap" -inputs = [] -source = """ -. = [{"message": "hello"}, {"message": "world"}] -""" +```yaml +transforms: + remap: + type: "remap" + inputs: [] + source: | + . = [{"message": "hello"}, {"message": "world"}] ``` Would generate two output events: @@ -45,15 +45,15 @@ Additionally, to make it easier to convert an incoming log event into an array, function to VRL that transforms an incoming event where one of the fields is an array into an array of events, each with one of the elements from the array field. This is easiest to see with an example: -```toml -[transforms.remap] -type = "remap" -inputs = [] -source = """ -. = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event +```yaml +transforms: + remap: + type: "remap" + inputs: [] + source: | + . = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event -. = unnest(.events) -""" + . = unnest(.events) ``` Would output the following log events: @@ -69,24 +69,23 @@ and another `remap` transform to receive each new event. An example of this: -```toml -[transforms.explode] -type = "remap" -inputs = [] -source = """ -. = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event - -. = unnest(.events) -""" - -[transforms.map] -type = "remap" -inputs = ["explode"] -source = """ -# example of pulling up the nested field to merge it into the top-level -. |= .events -del(.events) -""" +```yaml +transforms: + explode: + type: "remap" + inputs: [] + source: | + . = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event + + . = unnest(.events) + + map: + type: "remap" + inputs: ["explode"] + source: | + # example of pulling up the nested field to merge it into the top-level + . |= .events + del(.events) ``` Would output the following log events: diff --git a/website/content/en/highlights/2021-08-20-rate-limits.md b/website/content/en/highlights/2021-08-20-rate-limits.md index 5be8d63e4fa41..5668116faa5da 100644 --- a/website/content/en/highlights/2021-08-20-rate-limits.md +++ b/website/content/en/highlights/2021-08-20-rate-limits.md @@ -23,9 +23,10 @@ have no rate limiting and a maximum number of concurrent requests of 1024. To configure a request rate limit or maximum concurrency on a HTTP-based sink, you can set the `request` parameters like: -```toml -request.concurrency = 5 # limit to 5 in-flight requests -request.rate_limit_num = 10 # limit to 10 requests / second +```yaml +request: + concurrency: 5 # limit to 5 in-flight requests + rate_limit_num: 10 # limit to 10 requests / second ``` If you haven't already, we recommend trying out our [adaptive concurrency diff --git a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md index 96c30410f344c..5a33295331e6c 100644 --- a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md +++ b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md @@ -30,11 +30,12 @@ components. For example, with the component config: -```toml -[transforms.parse_nginx] -type = "remap" -inputs = [] -source = "" +```yaml +transforms: + parse_nginx: + type: "remap" + inputs: [] + source: "" ``` The `parse_nginx` part of the config is now only referred to as `ID` in the documentation. @@ -57,19 +58,22 @@ limitations and is only useful for backward compatibility with older clients. We no longer allow you to set the encoding of the payloads in the Datadog logs sink. For instance, if your configuration looks like so: -```toml -[sinks.dd_logs_egress] -type = "datadog_logs" -inputs = ["datadog_agent"] -encoding.codec = "json" +```yaml +sinks: + dd_logs_egress: + type: "datadog_logs" + inputs: ["datadog_agent"] + encoding: + codec: "json" ``` You should remove `encoding.codec` entirely, leaving you with: -```toml -[sinks.dd_logs_egress] -type = "datadog_logs" -inputs = ["datadog_agent"] +```yaml +sinks: + dd_logs_egress: + type: "datadog_logs" + inputs: ["datadog_agent"] ``` Encoding fields other than `codec` are still valid. diff --git a/website/content/en/highlights/2021-10-06-arc-default.md b/website/content/en/highlights/2021-10-06-arc-default.md index 54cfb15f834be..49344701a0eb3 100644 --- a/website/content/en/highlights/2021-10-06-arc-default.md +++ b/website/content/en/highlights/2021-10-06-arc-default.md @@ -20,8 +20,9 @@ increased pressure downstream, to avoid overwhelming the sink destination. This feature was previously able to be opted into with the following configuration: -```toml -request.concurrency = "adaptive" +```yaml +request: + concurrency: "adaptive" ``` This is the new default for HTTP-based sinks. As mentioned in the [announcement @@ -35,8 +36,9 @@ If you would like to instead use a fixed concurrency as was previously the default, you can set a static value like: -```toml -request.concurrency = 5 +```yaml +request: + concurrency: 5 ``` This will tell Vector to limit to 5 concurrent requests. diff --git a/website/content/en/highlights/2021-10-06-source-codecs.md b/website/content/en/highlights/2021-10-06-source-codecs.md index 09967fc40923f..7b4b307c5b923 100644 --- a/website/content/en/highlights/2021-10-06-source-codecs.md +++ b/website/content/en/highlights/2021-10-06-source-codecs.md @@ -17,12 +17,14 @@ added new `decoding` options to [most sources][9404]. For example, if you have a `kafka` source that has JSON-encoded messages, now you can simply add `decoding.codec = "json"` to your source configuration like: -```toml -[sources.kafka] -type = "kafka" -bootstrap_servers = "localhost:9200" -topics = ["my_topic"] -decoding.codec = "json" +```yaml +sources: + kafka: + type: "kafka" + bootstrap_servers: "localhost:9200" + topics: ["my_topic"] + decoding: + codec: "json" ``` This will decode your messages from JSON, thus saving you from an additional @@ -35,12 +37,15 @@ separating messages). For example, if you have an `http` source where the messages are delimited by commas instead of newlines, you can configure this like: -```toml -[sources.http] -type = "http" -address = "0.0.0.0:8080" -framing.method = "character_delimited" -framing.character_delimited.delimiter = "," +```yaml +sources: + http: + type: "http" + address: "0.0.0.0:8080" + framing: + method: "character_delimited" + character_delimited: + delimiter: "," ``` To have Vector parse each comma-delimited element as a new message. This can be diff --git a/website/content/en/highlights/2021-11-12-event-throttle-transform.md b/website/content/en/highlights/2021-11-12-event-throttle-transform.md index 4f029774a0e84..c6e51d7307b9f 100644 --- a/website/content/en/highlights/2021-11-12-event-throttle-transform.md +++ b/website/content/en/highlights/2021-11-12-event-throttle-transform.md @@ -48,12 +48,13 @@ Given these incoming log events: ...and this configuration... -```toml -[transforms.my_transform_id] -type = "throttle" -inputs = [ "my-source-or-transform-id" ] -threshold = 1 -window_secs = 60 +```yaml +transforms: + my_transform_id: + type: "throttle" + inputs: ["my-source-or-transform-id"] + threshold: 1 + window_secs: 60 ``` ...only one event will be allowed through: diff --git a/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md b/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md index a4e4d0a87b766..54b5f2b85e1e3 100644 --- a/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md +++ b/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md @@ -31,8 +31,8 @@ would not work with Splunk senders that required them. Now, you can configure the `splunk_hec` source to use the indexer acknowledgements protocol by configuring: -```toml -acknowledgements = true +```yaml +acknowledgements: true ``` When enabled, responses to incoming requests will include an ID that can be used to query for acknowledgement status at the newly exposed `/services/collector/ack` endpoint ([learn more here][indexer how it works]). The acknowledgement status is wired into Vector's @@ -51,8 +51,9 @@ acknowledgements][indexer] part of the HEC protocol. This has defaulted to on to provide higher guarantees, but can be disabled to restore the previous behavior by configuring: -```toml -acknowledgements.indexer_acknowledgements_enabled = false +```yaml +acknowledgements: + indexer_acknowledgements_enabled: false ``` ## Passthrough Token Routing diff --git a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md index e23e7b54c9f5e..7655ac4027eb9 100644 --- a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md +++ b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md @@ -165,11 +165,11 @@ source, you will now see a `splunk_channel` field in events output from the included channel information from requests in their events. If you want to remove this field, you can simply drop it with a `remap` transform like follows: -```toml -[transforms.foo] -type = "remap" -inputs = ["bar"] # where bar is the splunk_hec source -source = ''' - del(.splunk_channel) -''' +```yaml +transforms: + foo: + type: "remap" + inputs: ["bar"] # where bar is the splunk_hec source + source: | + del(.splunk_channel) ``` diff --git a/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md b/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md index 9444f10cbb47a..be461a4bc60e2 100644 --- a/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md +++ b/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md @@ -13,30 +13,28 @@ full support for testing task-style transforms (previously there may have been issues when using multiple inputs with task transforms). For example, you can now test `remap` transform's `dropped` output like so, -```toml -[transforms.foo] - type = "remap" - inputs = [] - drop_on_abort = true - reroute_dropped = true - source = "abort" +```yaml +transforms: + foo: + type: "remap" + inputs: [] + drop_on_abort: true + reroute_dropped: true + source: "abort" -[[tests]] - name = "remap_dropped_output" - no_outputs_from = [ "foo" ] - - [[tests.inputs]] - insert_at = "foo" - type = "log" - [tests.inputs.log_fields] - message = "I will be dropped" - - [[tests.outputs]] - extract_from = "foo.dropped" - - [[tests.outputs.conditions]] - type = "vrl" - source = 'assert_eq!(.message, "I will be dropped", "incorrect message")' +tests: + - name: "remap_dropped_output" + no_outputs_from: ["foo"] + inputs: + - insert_at: "foo" + type: "log" + log_fields: + message: "I will be dropped" + outputs: + - extract_from: "foo.dropped" + conditions: + - type: "vrl" + source: 'assert_eq!(.message, "I will be dropped", "incorrect message")' ``` Under-the-hood, we've reworked the unit testing implementation to more closely diff --git a/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md b/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md index 560c5389473cc..54c118ea395ff 100644 --- a/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md +++ b/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md @@ -33,10 +33,12 @@ persistence, regardless of issues with Vector and the host system. You may already be familiar with disk buffers if you have a sink configuration that looks like this: -```toml -[sinks.http] -# ... -buffer.type = "disk" +```yaml +sinks: + http: + # ... + buffer: + type: "disk" ``` ## Current disk buffer implementation @@ -74,16 +76,20 @@ for those who don't utilize a stateless deployment process as mentioned above. With all of that said, changing Vector to use the new disk buffer implementation is as simple as changing a single line: -```toml +```yaml # From this: -[sinks.http] -# ... -buffer.type = "disk" +sinks: + http: + # ... + buffer: + type: "disk" # To this: -[sinks.http] -# ... -buffer.type = "disk_v2" +sinks: + http: + # ... + buffer: + type: "disk_v2" ``` ## Let us know what you think! diff --git a/website/content/en/highlights/2022-03-15-vrl-vm-beta.md b/website/content/en/highlights/2022-03-15-vrl-vm-beta.md index f4bd959f259c6..399782a428435 100644 --- a/website/content/en/highlights/2022-03-15-vrl-vm-beta.md +++ b/website/content/en/highlights/2022-03-15-vrl-vm-beta.md @@ -20,14 +20,14 @@ the VRL program that is being run. You can give the VM a try by adding the `runtime` configuration option to your Remap transform: -```toml -[transforms.remap] -type = "remap" -inputs = [ "..." ] -runtime = "vm" -source = ''' -... -''' +```yaml +transforms: + remap: + type: "remap" + inputs: ["..."] + runtime: "vm" + source: | + ... ``` ## Let us know what you think! diff --git a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md index 761e6d861455b..96714e842b4ce 100644 --- a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md @@ -64,24 +64,28 @@ Here are some examples that require migrating | foo\\"bar | "foo\\"bar" | Double quotes and backlashes must be escaped _inside_ quotes | -### TOML transform example +### YAML transform example Old syntax -```toml -[transforms.dedupe] -type = "dedupe" -inputs = ["input"] -fields.match = ["message.user-identifier"] +```yaml +transforms: + dedupe: + type: "dedupe" + inputs: ["input"] + fields: + match: ["message.user-identifier"] ``` New syntax (the dash requires the field name to be quoted) -```toml -[transforms.dedupe] -type = "dedupe" -inputs = ["input"] -fields.match = ["message.\"user-identifier\""] +```yaml +transforms: + dedupe: + type: "dedupe" + inputs: ["input"] + fields: + match: ['message."user-identifier"'] ``` For more information on the new syntax, you can review the [VRL path expressions documentation](https://vector.dev/docs/reference/vrl/expressions/#path) diff --git a/website/content/en/highlights/2022-03-31-native-event-codecs.md b/website/content/en/highlights/2022-03-31-native-event-codecs.md index 0a10938afffea..1f1b301caca82 100644 --- a/website/content/en/highlights/2022-03-31-native-event-codecs.md +++ b/website/content/en/highlights/2022-03-31-native-event-codecs.md @@ -23,12 +23,14 @@ transform. For example, an `exec` source can now be configured to receive events via the `native_json` codec: -```toml -[sources.in] -type = "exec" -mode = "scheduled" -command = ["./scrape.sh"] -decoding.codec = "native_json" +```yaml +sources: + in: + type: "exec" + mode: "scheduled" + command: ["./scrape.sh"] + decoding: + codec: "native_json" ``` If `scrape.sh` contained: @@ -82,24 +84,28 @@ deserialize directly to the same native representation within Vector. Example source configuration: -```toml -[sources.in] -type = "kafka" -bootstrap_servers = "localhost:9092" -topics = ["vector"] -decoding.codec = "native" +```yaml +sources: + in: + type: "kafka" + bootstrap_servers: "localhost:9092" + topics: ["vector"] + decoding: + codec: "native" ``` This would allow an instance of Vector to receive events from another Vector instance that has a `kafka` sink configured like: -```toml -[sinks.out] -type = "kafka" -inputs = ["..."] -bootstrap_servers = "localhost:9092" -topic = "vector" -encoding.codec = "native" +```yaml +sinks: + out: + type: "kafka" + inputs: ["..."] + bootstrap_servers: "localhost:9092" + topic: "vector" + encoding: + codec: "native" ``` Note that the new `native` encoding option is not yet documented on sinks as we diff --git a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md index b7be9fad5dd50..f9a18b8e47cba 100644 --- a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md @@ -32,40 +32,39 @@ The `gcp_stackdriver_metrics` sink now matches the `gcp_stackdriver_logs` configuration, and doesn't require an additional `labels` section to add labels to submitted metrics. -##### TOML transform example +##### YAML transform example Old configuration -```toml -[sinks.my_sink_id] -type = "gcp_stackdriver_metrics" -inputs = [ "my-source-or-transform-id" ] -credentials_path = "/path/to/credentials.json" -project_id = "vector-123456" - - [sinks.my_sink_id.resource] - type = "global" - - [sinks.my_sink_id.resource.labels] - projectId = "vector-123456" - instanceId = "Twilight" - zone = "us-central1-a" +```yaml +sinks: + my_sink_id: + type: "gcp_stackdriver_metrics" + inputs: ["my-source-or-transform-id"] + credentials_path: "/path/to/credentials.json" + project_id: "vector-123456" + resource: + type: "global" + labels: + projectId: "vector-123456" + instanceId: "Twilight" + zone: "us-central1-a" ``` New configuration -```toml -[sinks.my_sink_id] -type = "gcp_stackdriver_metrics" -inputs = [ "my-source-or-transform-id" ] -credentials_path = "/path/to/credentials.json" -project_id = "vector-123456" - - [sinks.my_sink_id.resource] - type = "global" - projectId = "vector-123456" - instanceId = "Twilight" - zone = "us-central1-a" +```yaml +sinks: + my_sink_id: + type: "gcp_stackdriver_metrics" + inputs: ["my-source-or-transform-id"] + credentials_path: "/path/to/credentials.json" + project_id: "vector-123456" + resource: + type: "global" + projectId: "vector-123456" + instanceId: "Twilight" + zone: "us-central1-a" ``` For more information on the new syntax, you can review the [GCP Stackdriver Metrics sink documentation](https://vector.dev/docs/reference/configuration/sinks/gcp_stackdriver_metrics/) @@ -179,413 +178,442 @@ transform. Before: -```toml -[transforms.add_fields] -type = "add_fields" -inputs = ["some_input"] -fields.parent.child2 = "value2" +```yaml +transforms: + add_fields: + type: "add_fields" + inputs: ["some_input"] + fields: + parent: + child2: "value2" ``` After: -```toml -[transforms.add_fields] -type = "remap" -inputs = ["some_input"] -source = ''' -.parent.child2 = "value2" -''' +```yaml +transforms: + add_fields: + type: "remap" + inputs: ["some_input"] + source: | + .parent.child2 = "value2" ``` ##### `add_tags` Before: -```toml -[transforms.add_tags] -type = "add_tags" -inputs = ["some_input"] -tags.some_tag = "some_value" +```yaml +transforms: + add_tags: + type: "add_tags" + inputs: ["some_input"] + tags: + some_tag: "some_value" ``` After: -```toml -[transforms.add_tags] -type = "remap" -inputs = ["some_input"] -source = ''' -.tags.some_tag = "some_value" -''' +```yaml +transforms: + add_tags: + type: "remap" + inputs: ["some_input"] + source: | + .tags.some_tag = "some_value" ``` ##### `ansi_stripper` Before: -```toml -[transforms.ansi_stripper] -type = "ansi_stripper" -inputs = ["some_input"] +```yaml +transforms: + ansi_stripper: + type: "ansi_stripper" + inputs: ["some_input"] ``` After: -```toml -[transforms.ansi_stripper] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -.message = strip_ansi_escape_codes(string!(.message)) -''' +```yaml +transforms: + ansi_stripper: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + .message = strip_ansi_escape_codes(string!(.message)) ``` ##### `aws_cloudwatch_logs_subscription_parser` Before: -```toml -[transforms.aws_cloudwatch_logs_subscription_parser] -type = "aws_cloudwatch_logs_subscription_parser" -inputs = ["some_input"] +```yaml +transforms: + aws_cloudwatch_logs_subscription_parser: + type: "aws_cloudwatch_logs_subscription_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.aws_cloudwatch_logs_subscription_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_aws_cloudwatch_log_subscription_message!(.message) -''' +```yaml +transforms: + aws_cloudwatch_logs_subscription_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_aws_cloudwatch_log_subscription_message!(.message) ``` ##### `coercer` Before: -```toml -[transforms.coercer] -type = "coercer" -inputs = ["some_input"] -types.some_bool = "bool" -types.some_float = "float" -types.some_int = "int" -types.some_string = "string" -types.some_timestamp = "timestamp" +```yaml +transforms: + coercer: + type: "coercer" + inputs: ["some_input"] + types: + some_bool: "bool" + some_float: "float" + some_int: "int" + some_string: "string" + some_timestamp: "timestamp" ``` After: -```toml -[transforms.coercer] -type = "remap" -inputs = ["some_input"] -source = ''' -.some_bool = to_bool!(.some_bool) -.some_float = to_float!(.some_float) -.some_int = to_int!(.some_int) -.some_string = to_string!(.some_string) -.some_timestamp = to_timestamp!(.some_timestamp) -''' +```yaml +transforms: + coercer: + type: "remap" + inputs: ["some_input"] + source: | + .some_bool = to_bool!(.some_bool) + .some_float = to_float!(.some_float) + .some_int = to_int!(.some_int) + .some_string = to_string!(.some_string) + .some_timestamp = to_timestamp!(.some_timestamp) ``` ##### `concat` Before: -```toml -[transforms.concat] -type = "concat" -inputs = ["some_input"] -items = ["month", "day", "year"] -target = "date" -joiner = "/" +```yaml +transforms: + concat: + type: "concat" + inputs: ["some_input"] + items: ["month", "day", "year"] + target: "date" + joiner: "/" ``` After: -```toml -[transforms.concat] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -.date = join!([.month, .day, .year], "/") -''' +```yaml +transforms: + concat: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + .date = join!([.month, .day, .year], "/") ``` ##### `grok_parser` Before: -```toml -[transforms.grok_parser] -type = "grok_parser" -inputs = ["some_input"] -pattern = "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}" -types.timestamp = "timestamp|%+" -types.level = "string" -types.message = "string" +```yaml +transforms: + grok_parser: + type: "grok_parser" + inputs: ["some_input"] + pattern: "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}" + types: + timestamp: "timestamp|%+" + level: "string" + message: "string" ``` After: -```toml -[transforms.grok_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_grok!(.message, "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}") -.timestamp = parse_timestamp!(.timestamp , format: "%+") -''' +```yaml +transforms: + grok_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_grok!(.message, "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}") + .timestamp = parse_timestamp!(.timestamp , format: "%+") ``` ##### `json_parser` Before: -```toml -[transforms.json_parser] -type = "json_parser" -inputs = ["some_input"] +```yaml +transforms: + json_parser: + type: "json_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.json_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= object!(parse_json(.message)) -''' +```yaml +transforms: + json_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= object!(parse_json(.message)) ``` ##### `key_value_parser` Before: -```toml -[transforms.key_value_parser] -type = "key_value_parser" -inputs = ["some_input"] +```yaml +transforms: + key_value_parser: + type: "key_value_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.key_value_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_key_value!(.message) -''' +```yaml +transforms: + key_value_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_key_value!(.message) ``` ##### `logfmt_parser` Before: -```toml -[transforms.logfmt_parser] -type = "logfmt_parser" -inputs = ["some_input"] +```yaml +transforms: + logfmt_parser: + type: "logfmt_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.logfmt_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_logfmt!(.message) -''' +```yaml +transforms: + logfmt_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_logfmt!(.message) ``` ##### `merge` Before: -```toml -[transforms.merge] -type = "merge" -inputs = ["some_input"] +```yaml +transforms: + merge: + type: "merge" + inputs: ["some_input"] ``` After: -```toml -[transforms.merge] -type = "reduce" -inputs = ["some_input"] -starts_when = "._partial == true" -merge_strategies.message = "concat" +```yaml +transforms: + merge: + type: "reduce" + inputs: ["some_input"] + starts_when: "._partial == true" + merge_strategies: + message: "concat" ``` ##### `regex_parser` Before: -```toml -[transforms.regex_parser] -type = "regex_parser" -inputs = ["some_input"] -patterns = ['^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$'] -types.bytes_in = "int" -types.timestamp = "timestamp|%d/%m/%Y:%H:%M:%S %z" -types.status = "int" -types.bytes_out = "int" +```yaml +transforms: + regex_parser: + type: "regex_parser" + inputs: ["some_input"] + patterns: + - '^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$' + types: + bytes_in: "int" + timestamp: "timestamp|%d/%m/%Y:%H:%M:%S %z" + status: "int" + bytes_out: "int" ``` After: -```toml -[transforms.regex_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_regex!(.message, [#"^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$"#] -.bytes_in = to_int!(.bytes_in) -.some_timestamp = parse_timestamp!(.some_timestamp, "%d/%m/%Y:%H:%M:%S %z") -.status = to_int!(.status) -.bytes_out = to_int!(.bytes_out) -''' +```yaml +transforms: + regex_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_regex!(.message, [#"^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$"#] + .bytes_in = to_int!(.bytes_in) + .some_timestamp = parse_timestamp!(.some_timestamp, "%d/%m/%Y:%H:%M:%S %z") + .status = to_int!(.status) + .bytes_out = to_int!(.bytes_out) ``` ##### `remove_fields` Before: -```toml -[transforms.remove_fields] -type = "remove_fields" -inputs = ["some_input"] -fields = ["parent.child"] +```yaml +transforms: + remove_fields: + type: "remove_fields" + inputs: ["some_input"] + fields: ["parent.child"] ``` After: -```toml -[transforms.remove_fields] -type = "remap" -inputs = ["some_input"] -source = ''' -del(.parent.child) -''' +```yaml +transforms: + remove_fields: + type: "remap" + inputs: ["some_input"] + source: | + del(.parent.child) ``` ##### `remove_tags` Before: -```toml -[transforms.remove_tags] -type = "remove_tags" -inputs = ["some_input"] -tags = ["some_tag"] +```yaml +transforms: + remove_tags: + type: "remove_tags" + inputs: ["some_input"] + tags: ["some_tag"] ``` After: -```toml -[transforms.remove_tags] -type = "remap" -inputs = ["some_input"] -source = ''' -del(.tags.some_tag) -''' +```yaml +transforms: + remove_tags: + type: "remap" + inputs: ["some_input"] + source: | + del(.tags.some_tag) ``` ##### `rename_fields` Before: -```toml -[transforms.rename_fields] -type = "rename_fields" -inputs = ["some_input"] -fields.new_name = ["old_name"] +```yaml +transforms: + rename_fields: + type: "rename_fields" + inputs: ["some_input"] + fields: + new_name: ["old_name"] ``` After: -```toml -[transforms.rename_fields] -type = "remap" -inputs = ["some_input"] -source = ''' -.new_name = del(.old_name) -''' +```yaml +transforms: + rename_fields: + type: "remap" + inputs: ["some_input"] + source: | + .new_name = del(.old_name) ``` ##### `split` Before: -```toml -[transforms.split] -type = "split" -inputs = ["some_input"] -field_names = ["remote_addr", "user_id", "timestamp", "message", "status", "bytes"] -types.status = "int" -types.bytes = "int" +```yaml +transforms: + split: + type: "split" + inputs: ["some_input"] + field_names: ["remote_addr", "user_id", "timestamp", "message", "status", "bytes"] + types: + status: "int" + bytes: "int" ``` After: -```toml -[transforms.split] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -values = split(.message) -.remote_addr = values[0] -.user_id = values[1] -.timestamp = values[2] -.message = values[3] -.status = to_int!(values[4]) -.bytes = to_int!(values[5]) -''' +```yaml +transforms: + split: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + values = split(.message) + .remote_addr = values[0] + .user_id = values[1] + .timestamp = values[2] + .message = values[3] + .status = to_int!(values[4]) + .bytes = to_int!(values[5]) ``` ##### `tokenizer` Before: -```toml -[transforms.tokenizer] -type = "tokenizer" -inputs = ["some_input"] -field_names = ["remote_addr", "ident", "user_id", "timestamp", "message", "status", "bytes"] -.types.status = "int" -.types.bytes = "int" +```yaml +transforms: + tokenizer: + type: "tokenizer" + inputs: ["some_input"] + field_names: ["remote_addr", "ident", "user_id", "timestamp", "message", "status", "bytes"] + types: + status: "int" + bytes: "int" ``` After: -```toml -[transforms.tokenizer] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -values = parse_tokens!(.message) -.remote_addr = values[0] -.user_id = values[1] -.timestamp = values[2] -.message = values[3] -.status = to_int!(values[4]) -.bytes = to_int!(values[5]) -''' +```yaml +transforms: + tokenizer: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + values = parse_tokens!(.message) + .remote_addr = values[0] + .user_id = values[1] + .timestamp = values[2] + .message = values[3] + .status = to_int!(values[4]) + .bytes = to_int!(values[5]) ``` diff --git a/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md b/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md index 7fb2d3003977c..b71453f2e9cd1 100644 --- a/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md @@ -135,15 +135,16 @@ will give a compile-time error if you try to mutate the event in a condition. Example filter transform config -```toml -[transforms.filter] -type = "filter" -inputs = [ "input" ] -condition.type = "vrl" -condition.source = """ -.foo = "bar" -true -""" +```yaml +transforms: + filter: + type: "filter" + inputs: ["input"] + condition: + type: "vrl" + source: | + .foo = "bar" + true ``` New error @@ -253,8 +254,9 @@ need to remove it to avoid Vector from erroring on startup. The `gcp_pubsub` sink now supports a variety of codecs. To encode your logs as JSON before publishing them to Cloud Pub/Sub, add the following encoding option -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` to the config of your `gcp_pubsub` sink. @@ -264,8 +266,9 @@ to the config of your `gcp_pubsub` sink. The `humio_metrics` sink configuration no longer expects an `encoding` option. If you previously used the encoding option -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` you need to remove the line from your `humio_metrics` config. Metrics are now @@ -277,8 +280,9 @@ We streamlined the encoding configuration for our sinks, enabling all applicable from a variety of codecs: `json`, `text`, `raw`, `logfmt`, `avro`, `native` or `native_json`, e.g. by setting -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` in your sink configuration. @@ -287,8 +291,9 @@ Additionally, some sinks now support configuring how encoded events should be se stream or batch: `bytes`, `character_delimited`, `length_delimited` or `newline_delimited`, e.g. by setting -```toml -framing.method = "newline_delimited" +```yaml +framing: + method: "newline_delimited" ``` in your sink configuration. @@ -353,10 +358,9 @@ This was introduced in order to better handle metrics sent from the `datadog_age The result is that configurations with VRL expressions that expect the namespace to be in the name will need to be adapted to either remove the namespace from the name, or join the namespace and the name, for example: -```toml -full_metric_name = """ -join([.namespace, .name], ".") -""" +```yaml +full_metric_name: | + join([.namespace, .name], ".") ``` ### Deprecations @@ -365,14 +369,15 @@ join([.namespace, .name], ".") We are deprecating setting encoding options by a shorthand string. E.g. when your sink encoding used -```toml -encoding = "json" +```yaml +encoding: "json" ``` it should now be replaced by explicitly setting the `codec` -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` #### Sink encoding value `ndjson` is now `json` encoding + `newline_delimited` framing {#sink-encoding-ndjson-json} @@ -382,20 +387,24 @@ can be set explicitly via a dedicated `framing` option. This affects all sink configurations that previously used -```toml -encoding.codec = "ndjson" +```yaml +encoding: + codec: "ndjson" ``` The `http`, `aws_s3`, `gcp_cloud_storage` and `azure_blob` sinks should be configured to use a combination of `json` encoding and `newline_delimited` framing instead -```toml -framing.method = "newline_delimited" -encoding.codec = "json" +```yaml +framing: + method: "newline_delimited" +encoding: + codec: "json" ``` For all other sinks, simply set the codec to `json` to maintain the current behavior -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` diff --git a/website/content/en/highlights/2022-07-07-secrets-management.md b/website/content/en/highlights/2022-07-07-secrets-management.md index 5f69e3783df04..6fc766134e671 100644 --- a/website/content/en/highlights/2022-07-07-secrets-management.md +++ b/website/content/en/highlights/2022-07-07-secrets-management.md @@ -19,21 +19,24 @@ these values can be read if a user on a host has access to read from the `/proc` A secret backend can be configured like this: -```toml -[secret.backend_1] -type = "exec" # exec is the only supported backend as of writing -command = ["/path/to/cmd1"] +```yaml +secret: + backend_1: + type: "exec" # exec is the only supported backend as of writing + command: ["/path/to/cmd1"] ``` You can then specify where secrets should be read via `SECRET[.]` in the config like: -```toml -[sources.my_source_id] -type = "aws_sqs" -region = "us-east-1" -queue_url = "https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue" -auth.access_key_id = "SECRET[backend_1.aws_access_key_id]" -auth.secret_access_key = "SECRET[backend_1.aws_secret_access_key]" +```yaml +sources: + my_source_id: + type: "aws_sqs" + region: "us-east-1" + queue_url: "https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue" + auth: + access_key_id: "SECRET[backend_1.aws_access_key_id]" + secret_access_key: "SECRET[backend_1.aws_secret_access_key]" ``` Here `auth.access_key_id` and `auth.secret_access_key` will use secrets provided by the `backend_1` secret backend. diff --git a/website/content/en/highlights/2022-07-07-sink-codecs.md b/website/content/en/highlights/2022-07-07-sink-codecs.md index 7e00290826a50..d6c0f75c47020 100644 --- a/website/content/en/highlights/2022-07-07-sink-codecs.md +++ b/website/content/en/highlights/2022-07-07-sink-codecs.md @@ -20,13 +20,16 @@ For example, if you have a `socket` sink that you want to send [length-delimited][length_delimited] JSON-encoded, messages, you can now do so with configuration like: -```toml -[sinks.socket] -type = "socket" -address = "92.12.333.224:5000" -mode = "tcp" -framing.method = "length_delimited" -encoding.codec = "json" +```yaml +sinks: + socket: + type: "socket" + address: "92.12.333.224:5000" + mode: "tcp" + framing: + method: "length_delimited" + encoding: + codec: "json" ``` This will encode messages flowing into this sink as JSON and frame them using diff --git a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md index 8464524b01c45..61050ecefca6b 100644 --- a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md @@ -67,16 +67,17 @@ Additionally, the [account ID][nr_account_id] must also now be specified. All put together, here's an example of converting from a `new_relic_logs` sink configuration over to the `new_relic` sink configuration: -```toml -[sinks.new_relic_logs] -type = "new_relic_logs" -license_key = "xxxx" - -[sinks.new_relic] -type = "new_relic" -license_key = "xxxx" -account_id = "yyyy" -api = "logs" +```yaml +sinks: + new_relic_logs: + type: "new_relic_logs" + license_key: "xxxx" + + new_relic: + type: "new_relic" + license_key: "xxxx" + account_id: "yyyy" + api: "logs" ``` [0-24-0-upgrade-guide]: https://vector.dev/highlights/2022-08-16-0-24-0-upgrade-guide/#deprecated-components @@ -143,27 +144,28 @@ The older version 1 API is now deprecated and will be removed in a future versio For example, the following partial configuration: -```toml -[transform.example] -type = "lua" -version = 1 -source = """ - event["a"] = "some value" - event["b"] = nil -""" +```yaml +transforms: + example: + type: "lua" + version: 1 + source: | + event["a"] = "some value" + event["b"] = nil ``` would need to be converted to the following: -```toml -[transform.example] -type = "lua" -version = 2 -hooks.process = """ - function (event, emit) - event.log.a = "some value" - event.log.b = nil - emit(event) - end -""" +```yaml +transforms: + example: + type: "lua" + version: 2 + hooks: + process: | + function (event, emit) + event.log.a = "some value" + event.log.b = nil + emit(event) + end ``` diff --git a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md index 5813dfc248a40..7160cfc402ca0 100644 --- a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md @@ -73,10 +73,11 @@ The new behavior is to use the default strategy based on the element type. ###### Config -```toml -group_by = [ "id" ] -merge_strategies.id = "discard" -merge_strategies."a.b[0]" = "array" +```yaml +group_by: ["id"] +merge_strategies: + id: "discard" + "a.b[0]": "array" ``` ###### Event 1 From 43f620435a353b048c40661f59a2d2d18f2419ed Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 13 Apr 2026 12:24:37 -0400 Subject: [PATCH 074/364] fix(observability): fix panic in allocation tracing dealloc path (#25136) * fix(observability): fix panic in allocation tracing when deallocating pre-tracking memory When `--allocation-tracing` is enabled at runtime, the custom allocator wraps every allocation with an extra byte to store the allocation group ID. Previously, allocations made before tracking was enabled used the original (unwrapped) layout. When those were later freed, `dealloc` read an out-of-bounds byte as the group ID, hitting `NonZeroU8::new_unchecked(0)` -- undefined behavior that recent Rust toolchains (>= ~1.78) turn into an abort in debug builds. Additionally, reentrant allocations (wrapped layout but tracing closure skipped) left the group ID header uninitialized, causing misattributed deallocations and skewed per-group memory accounting. Fix: always allocate with the wrapped layout, regardless of whether tracking is currently enabled. The group ID header byte is set to: - UNTRACKED (0): tracking was off at allocation time - UNTRACED (u8::MAX): tracking was on but the tracing closure was skipped due to reentrancy - A real group ID (1..254): normal traced allocation On deallocation, all paths free with the wrapped layout (which is always correct now). UNTRACKED and UNTRACED skip `trace_deallocation` to keep per-group accounting balanced. This eliminates: - The original panic/UB from `NonZero::new_unchecked(0)` - Layout mismatches for pre-tracking allocations (including realloc) - Accounting skew from uninitialized headers on reentrant allocations This bug has been latent since #15221 (Nov 2022) which introduced the runtime toggle. Co-Authored-By: Claude Opus 4.6 (1M context) * add authors * fix(observability): strengthen allocation tracing tests Harden test coverage based on code review feedback: - Assert exact ROOT group ID and trace counters in tracked alloc test - Add end-to-end reentrant allocation test that exercises the real reentrancy path via thread-local borrow - Remove redundant manual-sentinel test now covered by reentrant test - Rename sentinel collision test to reflect what it actually checks Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...ix_allocation_tracing_dealloc_panic.fix.md | 3 + .../allocator/tracing_allocator.rs | 202 ++++++++++++++++-- 2 files changed, 192 insertions(+), 13 deletions(-) create mode 100644 changelog.d/fix_allocation_tracing_dealloc_panic.fix.md diff --git a/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md b/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md new file mode 100644 index 0000000000000..732c836eac0e8 --- /dev/null +++ b/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md @@ -0,0 +1,3 @@ +Fixed a panic (abort) when running with `--allocation-tracing` in debug builds, caused by deallocating memory that was allocated before tracking was enabled. Also fixed per-group memory accounting skew for reentrant allocations whose tracing closure was skipped, which left the group ID header uninitialized and caused deallocations to be attributed to wrong groups. + +authors: pront diff --git a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs index 8d1eff3e36d4e..d0a2027eeb2e4 100644 --- a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs +++ b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs @@ -9,8 +9,24 @@ use super::{ }; use crate::internal_telemetry::allocations::TRACK_ALLOCATIONS; +/// Header value for allocations made while tracking is disabled. +/// On deallocation we free with the wrapped layout but skip tracing. +const UNTRACKED: u8 = 0; + +/// Header value for allocations whose tracing closure was skipped due to +/// reentrancy. `register()` never hands out `u8::MAX`, so this cannot +/// collide with a real group ID. On deallocation we free with the wrapped +/// layout but skip `trace_deallocation` to keep accounting balanced. +const UNTRACED: u8 = u8::MAX; + /// A tracing allocator that groups allocation events by groups. /// +/// Every allocation made through this allocator uses the "wrapped" layout: +/// the requested layout extended by one byte to store an allocation group +/// ID. This byte is always present regardless of whether tracking is +/// enabled, which guarantees that `dealloc` can always read a valid header +/// and free with the correct (wrapped) layout. +/// /// This allocator can only be used when specified via `#[global_allocator]`. pub struct GroupedTraceableAllocator { allocator: A, @@ -29,11 +45,6 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn alloc(&self, object_layout: Layout) -> *mut u8 { unsafe { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - return self.allocator.alloc(object_layout); - } - - // Allocate our wrapped layout and make sure the allocation succeeded. let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); let actual_ptr = self.allocator.alloc(actual_layout); if actual_ptr.is_null() { @@ -42,6 +53,16 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + group_id_ptr.write(UNTRACKED); + return actual_ptr; + } + + // Write the untraced sentinel so that `dealloc` always finds a + // valid header, even when the tracing closure below is skipped + // due to reentrancy. + group_id_ptr.write(UNTRACED); + let object_size = object_layout.size(); try_with_suspended_allocation_group( @@ -58,19 +79,19 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn dealloc(&self, object_ptr: *mut u8, object_layout: Layout) { unsafe { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - self.allocator.dealloc(object_ptr, object_layout); - return; - } - // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the - // requested layout, not the wrapped layout that was actually allocated. let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); - let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); - // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. + // Always free with the wrapped layout since all allocations + // (tracked or not) use it. self.allocator.dealloc(object_ptr, wrapped_layout); + // Skip tracing for untracked (tracking was off) and untraced + // (reentrant, closure skipped) allocations. + if raw_group_id == UNTRACKED || raw_group_id == UNTRACED { + return; + } + let object_size = object_layout.size(); let source_group_id = AllocationGroupId::from_raw(raw_group_id); @@ -96,3 +117,158 @@ fn get_wrapped_layout(object_layout: Layout) -> (Layout, usize) { (actual_layout.pad_to_align(), offset_to_group_id) } + +#[cfg(test)] +mod tests { + use std::{ + alloc::{GlobalAlloc, Layout, System}, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use serial_test::serial; + + use super::*; + use crate::internal_telemetry::allocations::allocator::{ + token::AllocationGroupId, tracer::Tracer, + }; + + /// RAII guard that enables `TRACK_ALLOCATIONS` on creation and + /// restores it to `false` on drop, ensuring cleanup even if the + /// test panics. + struct TrackingGuard; + + impl TrackingGuard { + fn enable() -> Self { + TRACK_ALLOCATIONS.store(true, Ordering::Release); + Self + } + } + + impl Drop for TrackingGuard { + fn drop(&mut self) { + TRACK_ALLOCATIONS.store(false, Ordering::Release); + } + } + + struct CountingTracer { + allocs: AtomicUsize, + deallocs: AtomicUsize, + } + + impl CountingTracer { + const fn new() -> Self { + Self { + allocs: AtomicUsize::new(0), + deallocs: AtomicUsize::new(0), + } + } + } + + impl Tracer for CountingTracer { + fn trace_allocation(&self, _size: usize, _group_id: AllocationGroupId) { + self.allocs.fetch_add(1, Ordering::Relaxed); + } + + fn trace_deallocation(&self, _size: usize, _source_group_id: AllocationGroupId) { + self.deallocs.fetch_add(1, Ordering::Relaxed); + } + } + + #[test] + fn sentinels_do_not_collide_with_root_id() { + assert_eq!(UNTRACED, u8::MAX); + assert_ne!(UNTRACED, AllocationGroupId::ROOT.as_raw()); + assert_ne!(UNTRACKED, AllocationGroupId::ROOT.as_raw()); + } + + /// Allocations made while tracking is disabled get UNTRACKED (0) in the + /// header. Deallocating them (whether tracking is on or off) must use + /// the wrapped layout and skip tracing. + #[test] + #[serial] + fn untracked_alloc_dealloc_skips_tracing() { + let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); + let layout = Layout::from_size_align(64, 8).unwrap(); + + // Tracking starts off (default state). Allocate while disabled. + let ptr = unsafe { allocator.alloc(layout) }; + assert!(!ptr.is_null()); + + // Header must be UNTRACKED. + let (_, offset) = get_wrapped_layout(layout); + let raw_id = unsafe { ptr.add(offset).cast::().read() }; + assert_eq!(raw_id, UNTRACKED); + + // Enable tracking, then dealloc -- must not panic, no trace events. + let _guard = TrackingGuard::enable(); + unsafe { allocator.dealloc(ptr, layout) }; + + assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); + assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); + } + + /// Tracked allocation: header is a valid non-zero, non-sentinel group + /// ID, tracing fires for both alloc and dealloc, and dealloc completes + /// without panic. + #[test] + #[serial] + fn tracked_alloc_dealloc_does_not_panic() { + let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); + let layout = Layout::from_size_align(64, 8).unwrap(); + + let _guard = TrackingGuard::enable(); + let ptr = unsafe { allocator.alloc(layout) }; + assert!(!ptr.is_null()); + + let (_, offset) = get_wrapped_layout(layout); + let raw_id = unsafe { ptr.add(offset).cast::().read() }; + assert_eq!( + raw_id, + AllocationGroupId::ROOT.as_raw(), + "header must be the ROOT group ID" + ); + assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 1); + + unsafe { allocator.dealloc(ptr, layout) }; + + assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 1); + } + + /// End-to-end reentrant allocation: hold a mutable borrow on the + /// thread-local group stack so `try_with_suspended_allocation_group` + /// skips the tracing closure. The header must be UNTRACED and both + /// trace counters must stay at zero through alloc + dealloc. + #[test] + #[serial] + fn reentrant_alloc_writes_untraced_and_skips_tracing() { + use crate::internal_telemetry::allocations::allocator::token::LOCAL_ALLOCATION_GROUP_STACK; + + let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); + let layout = Layout::from_size_align(64, 8).unwrap(); + + let _guard = TrackingGuard::enable(); + + // Hold a mutable borrow to simulate reentrancy -- any nested + // `try_borrow_mut` inside `try_with_suspended_allocation_group` + // will fail, causing the tracing closure to be skipped. + LOCAL_ALLOCATION_GROUP_STACK.with(|group_stack| { + let _borrow = group_stack.borrow_mut(); + + let ptr = unsafe { allocator.alloc(layout) }; + assert!(!ptr.is_null()); + + let (_, offset) = get_wrapped_layout(layout); + let raw_id = unsafe { ptr.add(offset).cast::().read() }; + assert_eq!( + raw_id, UNTRACED, + "reentrant alloc must write UNTRACED sentinel" + ); + + assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); + + unsafe { allocator.dealloc(ptr, layout) }; + + assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); + }); + } +} From c6574fd66c328ef57fba27d2ad5422666c3e18eb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 13 Apr 2026 12:26:51 -0400 Subject: [PATCH 075/364] enhancement(api)!: replace custom Health RPC with standard gRPC health service (#25139) * enhancement(api)!: replace custom Health RPC with standard gRPC health service Replace the custom `ObservabilityService/Health` RPC with the standard `grpc.health.v1.Health` service on the observability API (port 8686). This enables native Kubernetes gRPC health probes, `grpc-health-probe`, and other standard tooling to work out of the box. The empty service name (`""`) is used for whole-server health, matching what Kubernetes probes and `grpc-health-probe` query by default. Key changes: - Remove Health RPC, HealthRequest, HealthResponse from the proto - Add tonic_health standard health service to the gRPC server - Switch vector-api-client to use tonic_health HealthClient; the client checks ServingStatus and returns NotServing error if not SERVING - Flip health to NOT_SERVING in TopologyController::stop before draining the topology, so Kubernetes removes the pod from endpoints early - Update docs, changelog, and add integration test Co-Authored-By: Claude Opus 4.6 (1M context) * fix gate * cleanup * Update src/topology/controller.rs Co-authored-by: Thomas --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Thomas --- Cargo.lock | 1 + Cargo.toml | 1 + changelog.d/graphql_to_grpc_api.breaking.md | 4 +- lib/vector-api-client/Cargo.toml | 1 + lib/vector-api-client/src/client.rs | 51 ++++++++++++++++----- lib/vector-api-client/src/error.rs | 3 ++ lib/vector-api-client/src/lib.rs | 6 +-- proto/vector/observability.proto | 11 +---- src/api/grpc/service.rs | 21 ++------- src/api/grpc_server.rs | 34 +++++++++----- src/app.rs | 3 -- src/topology/controller.rs | 21 +++++---- src/topology/running.rs | 9 +--- tests/vector_api/health.rs | 25 ++++++++++ tests/vector_api/lib.rs | 1 + website/content/en/docs/reference/api.md | 4 +- 16 files changed, 119 insertions(+), 77 deletions(-) create mode 100644 tests/vector_api/health.rs diff --git a/Cargo.lock b/Cargo.lock index 054d3afc66007..d013bc09432ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12453,6 +12453,7 @@ dependencies = [ "tokio-stream", "tonic 0.11.0", "tonic-build 0.11.0", + "tonic-health", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9389c9420d248..84d89c50096f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -561,6 +561,7 @@ docker = ["dep:bollard", "dep:dirs-next"] api = [ "dep:base64", "dep:tonic", + "dep:tonic-health", "dep:tonic-reflection", "dep:prost", "dep:prost-types", diff --git a/changelog.d/graphql_to_grpc_api.breaking.md b/changelog.d/graphql_to_grpc_api.breaking.md index 20ab9d67b35e3..2c9756ee16a89 100644 --- a/changelog.d/graphql_to_grpc_api.breaking.md +++ b/changelog.d/graphql_to_grpc_api.breaking.md @@ -27,8 +27,8 @@ vector top --url http://localhost:8686 [grpcurl](https://github.com/fullstorydev/grpcurl): ```bash -# Check health -grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/Health +# Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) +grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check # List components grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 00243fb9d0ecd..260a84a927e76 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -9,6 +9,7 @@ license = "MPL-2.0" [dependencies] # gRPC tonic.workspace = true +tonic-health.workspace = true prost.workspace = true prost-types.workspace = true diff --git a/lib/vector-api-client/src/client.rs b/lib/vector-api-client/src/client.rs index fcba2702a3e2a..0d8fda1b15aaf 100644 --- a/lib/vector-api-client/src/client.rs +++ b/lib/vector-api-client/src/client.rs @@ -1,17 +1,20 @@ use http::Uri; use tokio_stream::{Stream, StreamExt}; use tonic::transport::{Channel, Endpoint}; +use tonic_health::pb::{ + HealthCheckRequest, health_check_response::ServingStatus, health_client::HealthClient, +}; use crate::{ error::{Error, Result}, proto::{ GetAllocationTracingStatusRequest, GetAllocationTracingStatusResponse, - GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, - HealthRequest, HealthResponse, MetricName, StreamComponentAllocatedBytesRequest, - StreamComponentAllocatedBytesResponse, StreamComponentMetricsRequest, - StreamComponentMetricsResponse, StreamHeartbeatRequest, StreamHeartbeatResponse, - StreamOutputEventsRequest, StreamOutputEventsResponse, StreamUptimeRequest, - StreamUptimeResponse, observability_service_client::ObservabilityServiceClient, + GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, MetricName, + StreamComponentAllocatedBytesRequest, StreamComponentAllocatedBytesResponse, + StreamComponentMetricsRequest, StreamComponentMetricsResponse, StreamHeartbeatRequest, + StreamHeartbeatResponse, StreamOutputEventsRequest, StreamOutputEventsResponse, + StreamUptimeRequest, StreamUptimeResponse, + observability_service_client::ObservabilityServiceClient, }, }; @@ -19,6 +22,7 @@ use crate::{ #[derive(Debug, Clone)] pub struct Client { endpoint: Endpoint, + channel: Option, client: Option>, } @@ -33,6 +37,7 @@ impl Client { pub fn new(uri: Uri) -> Self { Self { endpoint: Endpoint::from(uri), + channel: None, client: None, } } @@ -40,7 +45,8 @@ impl Client { /// Connect to the gRPC server pub async fn connect(&mut self) -> Result<()> { let channel = self.endpoint.connect().await?; - self.client = Some(ObservabilityServiceClient::new(channel)); + self.client = Some(ObservabilityServiceClient::new(channel.clone())); + self.channel = Some(channel); Ok(()) } @@ -49,13 +55,34 @@ impl Client { self.client.as_mut().ok_or(Error::NotConnected) } + /// Get the underlying channel + fn channel(&self) -> Result<&Channel> { + self.channel.as_ref().ok_or(Error::NotConnected) + } + // ========== Unary RPCs ========== - /// Check if the API server is healthy - pub async fn health(&mut self) -> Result { - let client = self.ensure_connected()?; - let response = client.health(HealthRequest {}).await?; - Ok(response.into_inner()) + /// Check if the API server is healthy using the standard gRPC health check + /// protocol (grpc.health.v1.Health/Check). + /// + /// Queries the empty service name (`""`), which represents whole-server + /// health. This is the default used by Kubernetes gRPC probes and + /// `grpc-health-probe`. + /// + /// Returns `Ok(())` if the server is `SERVING`, or an error otherwise. + pub async fn health(&mut self) -> Result<()> { + let channel = self.channel()?.clone(); + let mut health_client = HealthClient::new(channel); + let response = health_client + .check(HealthCheckRequest { + service: String::new(), + }) + .await?; + let status = response.into_inner().status; + if status != ServingStatus::Serving as i32 { + return Err(Error::NotServing { status }); + } + Ok(()) } /// Get metadata about the Vector instance diff --git a/lib/vector-api-client/src/error.rs b/lib/vector-api-client/src/error.rs index 65fcfd80b511f..34eb347af12d8 100644 --- a/lib/vector-api-client/src/error.rs +++ b/lib/vector-api-client/src/error.rs @@ -16,6 +16,9 @@ pub enum Error { #[snafu(display("Not connected to gRPC server"))] NotConnected, + #[snafu(display("Server is not serving (status: {})", status))] + NotServing { status: i32 }, + #[snafu(display("Stream error: {}", message))] Stream { message: String }, } diff --git a/lib/vector-api-client/src/lib.rs b/lib/vector-api-client/src/lib.rs index a05002ebb3e61..74b72f2f0ea5b 100644 --- a/lib/vector-api-client/src/lib.rs +++ b/lib/vector-api-client/src/lib.rs @@ -11,9 +11,9 @@ //! let mut client = Client::new("http://localhost:9999".parse().unwrap()); //! client.connect().await?; //! -//! // Check health -//! let health = client.health().await?; -//! println!("Healthy: {}", health.healthy); +//! // Check health (standard gRPC health check) +//! client.health().await?; +//! println!("Server is healthy"); //! //! // Get components //! let components = client.get_components(0).await?; diff --git a/proto/vector/observability.proto b/proto/vector/observability.proto index 9996a47908949..57ce1a3f40002 100644 --- a/proto/vector/observability.proto +++ b/proto/vector/observability.proto @@ -8,9 +8,6 @@ import "event.proto"; service ObservabilityService { // ========== Simple Queries ========== - // Check if the API server is healthy and responding - rpc Health(HealthRequest) returns (HealthResponse); - // Get metadata about the Vector instance (version, hostname) rpc GetMeta(GetMetaRequest) returns (GetMetaResponse); @@ -42,13 +39,7 @@ service ObservabilityService { rpc StreamOutputEvents(StreamOutputEventsRequest) returns (stream StreamOutputEventsResponse); } -// ========== Health & Meta Messages ========== - -message HealthRequest {} - -message HealthResponse { - bool healthy = 1; -} +// ========== Meta Messages ========== message GetMetaRequest {} diff --git a/src/api/grpc/service.rs b/src/api/grpc/service.rs index 580ebb69b5219..110ef9d5ba441 100644 --- a/src/api/grpc/service.rs +++ b/src/api/grpc/service.rs @@ -4,10 +4,7 @@ use std::pin::Pin; // (only used in synchronous map updates inside IntervalStream closures), so the // cheaper std mutex is correct here. tokio::sync::Mutex is only needed when the // critical section itself contains .await. -use std::sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, -}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use futures::{StreamExt as FuturesStreamExt, stream}; @@ -399,12 +396,11 @@ fn ports_to_proto_outputs( /// gRPC observability service implementation. pub struct ObservabilityService { watch_rx: WatchRx, - running: Arc, } impl ObservabilityService { - pub const fn new(watch_rx: WatchRx, running: Arc) -> Self { - Self { watch_rx, running } + pub const fn new(watch_rx: WatchRx) -> Self { + Self { watch_rx } } } @@ -412,17 +408,6 @@ impl ObservabilityService { impl observability::Service for ObservabilityService { // ========== Simple Queries ========== - async fn health( - &self, - _request: Request, - ) -> Result, Status> { - if self.running.load(Ordering::Relaxed) { - Ok(Response::new(HealthResponse { healthy: true })) - } else { - Err(Status::unavailable("Vector is shutting down")) - } - } - async fn get_meta( &self, _request: Request, diff --git a/src/api/grpc_server.rs b/src/api/grpc_server.rs index 714459cd39b79..ce33b0da3f7d5 100644 --- a/src/api/grpc_server.rs +++ b/src/api/grpc_server.rs @@ -1,11 +1,8 @@ -use std::{ - error::Error as StdError, - net::SocketAddr, - sync::{Arc, atomic::AtomicBool}, -}; +use std::{error::Error as StdError, net::SocketAddr}; use tokio::sync::oneshot; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server as TonicServer; +use tonic_health::server::{HealthReporter, health_reporter}; use vector_lib::tap::topology::WatchRx; use super::grpc::ObservabilityService; @@ -14,6 +11,7 @@ use crate::{config::Config, proto::observability::Server as ObservabilityServer} /// gRPC API server for Vector observability. pub struct GrpcServer { _shutdown: oneshot::Sender<()>, + health_reporter: HealthReporter, addr: SocketAddr, } @@ -25,11 +23,7 @@ impl GrpcServer { /// is dropped. /// /// Returns an error if the server fails to bind to the configured address. - pub async fn start( - config: &Config, - watch_rx: WatchRx, - running: Arc, - ) -> crate::Result { + pub async fn start(config: &Config, watch_rx: WatchRx) -> crate::Result { let addr = config.api.address.ok_or_else(|| { crate::Error::from("API address not configured in config.api.address") })?; @@ -46,7 +40,11 @@ impl GrpcServer { info!("GRPC API server bound to {}.", actual_addr); - let service = ObservabilityService::new(watch_rx, running); + let service = ObservabilityService::new(watch_rx); + + // Create the standard gRPC health service (grpc.health.v1.Health). + // The empty service ("") is registered as SERVING by default. + let (health_reporter, health_service) = health_reporter(); let (_shutdown, rx) = oneshot::channel(); @@ -59,10 +57,12 @@ impl GrpcServer { .register_encoded_file_descriptor_set( crate::proto::observability::FILE_DESCRIPTOR_SET, ) + .register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET) .build() .expect("Failed to build reflection service"); let result = TonicServer::builder() + .add_service(health_service) .add_service(ObservabilityServer::new(service)) .add_service(reflection_service) .serve_with_incoming_shutdown(incoming, async { @@ -85,10 +85,22 @@ impl GrpcServer { Ok(Self { _shutdown, + health_reporter, addr: actual_addr, }) } + /// Signal that the server is no longer serving. + /// + /// Call this **before** draining the topology so that Kubernetes gRPC + /// readiness probes fail early and the pod is removed from endpoints + /// before the process exits. + pub async fn set_not_serving(&mut self) { + self.health_reporter + .set_service_status("", tonic_health::ServingStatus::NotServing) + .await; + } + /// Get the address the server is listening on pub const fn addr(&self) -> SocketAddr { self.addr diff --git a/src/app.rs b/src/app.rs index 904d42be11968..580c06642a6e3 100644 --- a/src/app.rs +++ b/src/app.rs @@ -38,8 +38,6 @@ use crate::{ }, trace, }; -#[cfg(feature = "api")] -use std::sync::Arc; static WORKER_THREADS: AtomicUsize = AtomicUsize::new(0); @@ -143,7 +141,6 @@ impl ApplicationConfig { let api_server = handle.block_on(api::GrpcServer::start( self.topology.config(), self.topology.watch(), - Arc::clone(&self.topology.running), )); match api_server { Ok(server) => { diff --git a/src/topology/controller.rs b/src/topology/controller.rs index b77629828a577..efbe82cf594ac 100644 --- a/src/topology/controller.rs +++ b/src/topology/controller.rs @@ -76,13 +76,7 @@ impl TopologyController { } else if self.api_server.is_none() { debug!("Starting gRPC API server."); - match api::GrpcServer::start( - self.topology.config(), - self.topology.watch(), - Arc::clone(&self.topology.running), - ) - .await - { + match api::GrpcServer::start(self.topology.config(), self.topology.watch()).await { Ok(api_server) => { let addr = api_server.addr(); info!( @@ -156,7 +150,18 @@ impl TopologyController { } } - pub async fn stop(self) { + #[cfg_attr(not(feature = "api"), allow(unused_mut))] + pub async fn stop(mut self) { + // Phase 1: Mark the gRPC API as unavailable so that external probes + // (e.g. Kubernetes readiness) fail early and stop routing traffic + // to this instance. + #[cfg(feature = "api")] + if let Some(server) = self.api_server.as_mut() { + server.set_not_serving().await; + } + + // Phase 2: Drain the topology -- shuts down sources, waits for + // in-flight events to flush through transforms and sinks. self.topology.stop().await; } diff --git a/src/topology/running.rs b/src/topology/running.rs index 6839379cd0f1a..a1ecf7278ae89 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -1,9 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - }, + sync::{Arc, Mutex}, }; use futures::{Future, FutureExt, future}; @@ -66,7 +63,6 @@ pub struct RunningTopology { pub(crate) config: Config, pub(crate) abort_tx: mpsc::UnboundedSender, watch: (WatchTx, WatchRx), - pub(crate) running: Arc, graceful_shutdown_duration: Option, utilization_registry: Option, utilization_task: Option, @@ -90,7 +86,6 @@ impl RunningTopology { tasks: HashMap::new(), abort_tx, watch: watch::channel(TapResource::default()), - running: Arc::new(AtomicBool::new(true)), graceful_shutdown_duration: config.graceful_shutdown_duration, config, utilization_registry: None, @@ -148,8 +143,6 @@ impl RunningTopology { /// dropped then everything from this RunningTopology instance is fully /// dropped. pub fn stop(self) -> impl Future { - // Update the API's health endpoint to signal shutdown - self.running.store(false, Ordering::Relaxed); // Create handy handles collections of all tasks for the subsequent // operations. let mut wait_handles = Vec::new(); diff --git a/tests/vector_api/health.rs b/tests/vector_api/health.rs new file mode 100644 index 0000000000000..a17d16eecf913 --- /dev/null +++ b/tests/vector_api/health.rs @@ -0,0 +1,25 @@ +//! Integration tests for the standard gRPC health check on the observability API. + +use super::{common::*, harness::*}; + +// ============================================================================ +// Tests +// ============================================================================ + +/// Verifies that the standard gRPC health check (empty service = whole-server +/// health) reports SERVING on a running Vector instance. +#[tokio::test] +async fn health_check_reports_serving() { + let config = single_source_config("demo", 1.0, None); + let mut harness = TestHarness::new(&config) + .await + .expect("Vector should start"); + + harness + .api_client() + .health() + .await + .expect("health check should report SERVING"); + + assert!(harness.check_running(), "Vector should still be running"); +} diff --git a/tests/vector_api/lib.rs b/tests/vector_api/lib.rs index e82f39d41b42e..c6c7fbec35dd6 100644 --- a/tests/vector_api/lib.rs +++ b/tests/vector_api/lib.rs @@ -5,5 +5,6 @@ mod common; mod harness; +mod health; mod tap; mod top; diff --git a/website/content/en/docs/reference/api.md b/website/content/en/docs/reference/api.md index 11a3566f6acfa..949353241929b 100644 --- a/website/content/en/docs/reference/api.md +++ b/website/content/en/docs/reference/api.md @@ -24,8 +24,8 @@ You can interact with it using any standard gRPC tooling. ### Example using grpcurl ```bash -# Check health -grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/Health +# Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) +grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check # List components grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents From 44637c3056c8850752e6f1c6aef8ab5ead810b0f Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 14 Apr 2026 11:04:29 -0400 Subject: [PATCH 076/364] chore(deps): bump datadog-ci from 5.12.0 to 5.13.0 (#25187) --- .../environment/npm-tools/package-lock.json | 2421 +---------------- scripts/environment/npm-tools/package.json | 2 +- 2 files changed, 145 insertions(+), 2278 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index 9e25b17830d18..d9e098652b6ad 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -1,332 +1,27 @@ { - "name": "environment", + "name": "npm-tools", "lockfileVersion": 3, "requires": true, "packages": { "": { "dependencies": { - "@datadog/datadog-ci": "5.12.0", + "@datadog/datadog-ci": "5.13.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } }, - "node_modules/@antfu/install-pkg": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", - "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", - "license": "MIT", - "dependencies": { - "package-manager-detector": "^1.3.0", - "tinyexec": "^1.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/@datadog/datadog-ci": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.12.0.tgz", - "integrity": "sha512-8zJOgx0ouVZtspXTGdx327Yt6UiRnaQSlMc7c343a1HaKC9nIlQhnd90G1frFZhopiqA1KD8h9CWBhygxAJW7w==", + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.13.0.tgz", + "integrity": "sha512-QEooWT2OE6J//OV4gka3F8Ca8QkmVGEyKR8zpJVZ6EufJVsNSnBX2V2VE3tzkvndcKRe2/Q9ZCiL9GUiZ4wTLA==", "license": "Apache-2.0", - "dependencies": { - "@datadog/datadog-ci-base": "5.12.0", - "@datadog/datadog-ci-plugin-coverage": "5.12.0", - "@datadog/datadog-ci-plugin-deployment": "5.12.0", - "@datadog/datadog-ci-plugin-dora": "5.12.0", - "@datadog/datadog-ci-plugin-gate": "5.12.0", - "@datadog/datadog-ci-plugin-junit": "5.12.0", - "@datadog/datadog-ci-plugin-sarif": "5.12.0", - "@datadog/datadog-ci-plugin-sbom": "5.12.0", - "clipanion": "3.2.1", - "typanion": "3.14.0" - }, "bin": { - "datadog-ci": "dist/cli.js" + "datadog-ci": "dist/bundle.js" }, "engines": { "node": ">=20" } }, - "node_modules/@datadog/datadog-ci-base": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.12.0.tgz", - "integrity": "sha512-ZyobD2nbzwRksWiYvcxl3ZuYE4+AX1is26qjWAs1ufhVyGBCx1cVqiDFEAILeyqkncZYIFGppjCyjUcxev+CBg==", - "license": "Apache-2.0", - "dependencies": { - "@antfu/install-pkg": "1.1.0", - "@types/datadog-metrics": "0.6.1", - "async-retry": "1.3.3", - "axios": "1.13.5", - "chalk": "3.0.0", - "clipanion": "3.2.1", - "datadog-metrics": "0.9.3", - "debug": "4.4.1", - "deep-extend": "0.6.0", - "fast-xml-parser": "5.5.9", - "form-data": "4.0.4", - "glob": "13.0.6", - "inquirer": "8.2.7", - "is-docker": "4.0.0", - "jest-diff": "30.2.0", - "js-yaml": "4.1.1", - "jszip": "3.10.1", - "proxy-agent": "6.4.0", - "semver": "7.6.3", - "simple-git": "3.33.0", - "terminal-link": "2.1.1", - "tiny-async-pool": "2.1.0", - "typanion": "3.14.0", - "upath": "2.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-plugin-aas": "5.12.0", - "@datadog/datadog-ci-plugin-cloud-run": "5.12.0", - "@datadog/datadog-ci-plugin-container-app": "5.12.0", - "@datadog/datadog-ci-plugin-coverage": "5.12.0", - "@datadog/datadog-ci-plugin-deployment": "5.12.0", - "@datadog/datadog-ci-plugin-dora": "5.12.0", - "@datadog/datadog-ci-plugin-gate": "5.12.0", - "@datadog/datadog-ci-plugin-junit": "5.12.0", - "@datadog/datadog-ci-plugin-lambda": "5.12.0", - "@datadog/datadog-ci-plugin-sarif": "5.12.0", - "@datadog/datadog-ci-plugin-sbom": "5.12.0", - "@datadog/datadog-ci-plugin-stepfunctions": "5.11.0", - "@datadog/datadog-ci-plugin-synthetics": "5.11.0", - "@datadog/datadog-ci-plugin-terraform": "5.11.0" - }, - "peerDependenciesMeta": { - "@datadog/datadog-ci-plugin-aas": { - "optional": true - }, - "@datadog/datadog-ci-plugin-cloud-run": { - "optional": true - }, - "@datadog/datadog-ci-plugin-container-app": { - "optional": true - }, - "@datadog/datadog-ci-plugin-coverage": { - "optional": true - }, - "@datadog/datadog-ci-plugin-deployment": { - "optional": true - }, - "@datadog/datadog-ci-plugin-dora": { - "optional": true - }, - "@datadog/datadog-ci-plugin-gate": { - "optional": true - }, - "@datadog/datadog-ci-plugin-junit": { - "optional": true - }, - "@datadog/datadog-ci-plugin-lambda": { - "optional": true - }, - "@datadog/datadog-ci-plugin-sarif": { - "optional": true - }, - "@datadog/datadog-ci-plugin-sbom": { - "optional": true - }, - "@datadog/datadog-ci-plugin-stepfunctions": { - "optional": true - }, - "@datadog/datadog-ci-plugin-synthetics": { - "optional": true - }, - "@datadog/datadog-ci-plugin-terraform": { - "optional": true - } - } - }, - "node_modules/@datadog/datadog-ci-base/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@datadog/datadog-ci-plugin-coverage": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.12.0.tgz", - "integrity": "sha512-ll4aBYF+FMOlOjAnNiPjO8iPmVB/qfBHrUOCk2EMnOaQn6Eve/FajCaSqeV2BoI9+3UXv04P6eH5nxPtpcM/cg==", - "license": "Apache-2.0", - "dependencies": { - "chalk": "3.0.0", - "fast-xml-parser": "5.5.9", - "form-data": "4.0.4", - "upath": "2.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-deployment": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.12.0.tgz", - "integrity": "sha512-6KDwUmdxwGxYoBEYqAY0uo7ocvtTq3p3cXW4KjpDXwZyr1wWstzXQxTx7qquZzMMNvsD6JCq+HDqwdTgaHSAog==", - "license": "Apache-2.0", - "dependencies": { - "axios": "1.13.5", - "chalk": "3.0.0", - "simple-git": "3.33.0" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-dora": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.12.0.tgz", - "integrity": "sha512-HrC6H5lLzRTv3tKiS2f5YMU/WWwgzMF2kHIJU6+pe0y/v1xUYCELV112XYldbg3dkENfpG/AwjUeO6B8rE3s+g==", - "license": "Apache-2.0", - "dependencies": { - "chalk": "3.0.0", - "simple-git": "3.33.0" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-gate": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.12.0.tgz", - "integrity": "sha512-YNZuRigmGDGK2tK6xRsaq0PPx/4CRVZ4U6q7xabIP+jmhoG7mbI1RFUjlXSnGfeU1SXm0bVQw8F69AcZCHtNOA==", - "license": "Apache-2.0", - "dependencies": { - "@types/uuid": "9.0.8", - "chalk": "3.0.0", - "uuid": "9.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-junit": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.12.0.tgz", - "integrity": "sha512-XPVULYU1RUN2kFmSWGfAgU+psGFhPu+5NxnHeAQubs/wYcGwdZUEAD2+tn4F3PxpCkTduEb4FO6mLpYbjBpuqA==", - "license": "Apache-2.0", - "dependencies": { - "chalk": "3.0.0", - "fast-xml-parser": "5.5.9", - "form-data": "4.0.4", - "upath": "2.0.1", - "uuid": "9.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-sarif": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.12.0.tgz", - "integrity": "sha512-tmUEv6sJWsFG9vnriSu9RZSrLvYo0EfvRaHiiO3ufxBC+NvU0HZYhtTqXVT6NnsUSVv/2FBnzl0hCnPDHhyBFg==", - "license": "Apache-2.0", - "dependencies": { - "ajv": "8.18.0", - "ajv-formats": "3.0.1", - "chalk": "3.0.0", - "form-data": "4.0.4", - "upath": "2.0.1", - "uuid": "9.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-sbom": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.12.0.tgz", - "integrity": "sha512-F0c0gfX4O+Qm1heLEhNCTl/tUPlVQvyFGYTUqJ0Z2niqT4Y3z7Oi6RSodgUSwGCKl6bwacRjTTdCrzI3V0n3nA==", - "license": "Apache-2.0", - "dependencies": { - "ajv": "8.18.0", - "ajv-formats": "3.0.1", - "axios": "1.13.5", - "chalk": "3.0.0", - "packageurl-js": "2.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@inquirer/external-editor": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", - "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", - "license": "MIT", - "dependencies": { - "chardet": "^2.1.1", - "iconv-lite": "^0.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@jest/diff-sequences": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", - "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/schemas": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", - "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.34.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@kwsites/file-exists": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", - "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", - "license": "MIT", - "dependencies": { - "debug": "^4.1.1" - } - }, - "node_modules/@kwsites/promise-deferred": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", - "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", - "license": "MIT" - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -362,12 +57,6 @@ "node": ">= 8" } }, - "node_modules/@sinclair/typebox": { - "version": "0.34.49", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", - "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", - "license": "MIT" - }, "node_modules/@sindresorhus/merge-streams": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", @@ -380,18 +69,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@tootallnate/quickjs-emscripten": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", - "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", - "license": "MIT" - }, - "node_modules/@types/datadog-metrics": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@types/datadog-metrics/-/datadog-metrics-0.6.1.tgz", - "integrity": "sha512-p6zVpfmNcXwtcXjgpz7do/fKyfndGhU5sGJVtb5Gn5PvLDiQUAgD0mI/itf/99sBi9DRxeyhFQ9dQF6OxxQNbA==", - "license": "MIT" - }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -419,733 +96,127 @@ "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", "license": "MIT" }, - "node_modules/@types/uuid": { - "version": "9.0.8", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", - "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", - "license": "MIT" + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, "engines": { - "node": ">= 14" + "node": ">=8" } }, - "node_modules/ajv": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", - "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, "funding": { "type": "github", - "url": "https://github.com/sponsors/epoberezkin" + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">= 12" } }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "color-convert": "^2.0.1" + "ms": "^2.1.3" }, "engines": { - "node": ">=8" + "node": ">=6.0" }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" - }, - "node_modules/ast-types": { - "version": "0.13.4", - "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", - "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", "license": "MIT", "dependencies": { - "tslib": "^2.0.1" + "character-entities": "^2.0.0" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "license": "MIT", - "dependencies": { - "retry": "0.13.1" + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.13.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", - "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.11", - "form-data": "^4.0.5", - "proxy-from-env": "^1.1.0" + "engines": { + "node": ">=6" } }, - "node_modules/axios/node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", "license": "MIT", "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "dequal": "^2.0.0" }, - "engines": { - "node": ">= 6" + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "license": "MIT", + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", "engines": { - "node": "18 || 20 || >=22" + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/basic-ftp": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.2.tgz", - "integrity": "sha512-1tDrzKsdCg70WGvbFss/ulVAxupNauGnOlgpyjKzeQxzyllBLS0CGLV7tjIXTK3ZQA9/FBEm9qyFFN1bciA6pw==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/bl/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/character-entities": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", - "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-legacy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", - "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-reference-invalid": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", - "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/chardet": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", - "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", - "license": "MIT" - }, - "node_modules/cli-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", - "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", - "license": "MIT", - "dependencies": { - "restore-cursor": "^3.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-spinners": { - "version": "2.9.2", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", - "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", - "license": "MIT", - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-width": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", - "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", - "license": "ISC", - "engines": { - "node": ">= 10" - } - }, - "node_modules/clipanion": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/clipanion/-/clipanion-3.2.1.tgz", - "integrity": "sha512-dYFdjLb7y1ajfxQopN05mylEpK9ZX0sO1/RfMXdfmwjlIsPkbh4p7A682x++zFPLDCo1x3p82dtljHf5cW2LKA==", - "license": "MIT", - "workspaces": [ - "website" - ], - "dependencies": { - "typanion": "^3.8.0" - }, - "peerDependencies": { - "typanion": "*" - } - }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "license": "MIT", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", - "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "license": "MIT" - }, - "node_modules/data-uri-to-buffer": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", - "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/datadog-metrics": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/datadog-metrics/-/datadog-metrics-0.9.3.tgz", - "integrity": "sha512-BVsBX2t+4yA3tHs7DnB5H01cHVNiGJ/bHA8y6JppJDyXG7s2DLm6JaozPGpgsgVGd42Is1CHRG/yMDQpt877Xg==", - "license": "MIT", - "dependencies": { - "debug": "3.1.0", - "dogapi": "2.8.4" - } - }, - "node_modules/datadog-metrics/node_modules/debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/datadog-metrics/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decode-named-character-reference": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", - "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", - "license": "MIT", - "dependencies": { - "character-entities": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "license": "MIT", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/defaults": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", - "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", - "license": "MIT", - "dependencies": { - "clone": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/degenerator": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", - "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", - "license": "MIT", - "dependencies": { - "ast-types": "^0.13.4", - "escodegen": "^2.1.0", - "esprima": "^4.0.1" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/dequal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", - "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/devlop": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", - "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", - "license": "MIT", - "dependencies": { - "dequal": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/dogapi": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/dogapi/-/dogapi-2.8.4.tgz", - "integrity": "sha512-065fsvu5dB0o4+ENtLjZILvXMClDNH/yA9H6L8nsdcNiz9l0Hzpn7aQaCOPYXxqyzq4CRPOdwkFXUjDOXfRGbg==", - "license": "MIT", - "dependencies": { - "extend": "^3.0.2", - "json-bigint": "^1.0.0", - "lodash": "^4.17.21", - "minimist": "^1.2.5", - "rc": "^1.2.8" - }, - "bin": { - "dogapi": "bin/dogapi" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/escodegen": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", - "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "license": "BSD-2-Clause", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -1162,57 +233,6 @@ "node": ">=8.6.0" } }, - "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.1.3" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.5.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.9.tgz", - "integrity": "sha512-jldvxr1MC6rtiZKgrFnDSvT8xuH+eJqxqOBThUVjYrxssYTo1avZLGql5l0a0BAERR01CadYzZ83kVEkbyDg+g==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "fast-xml-builder": "^1.1.4", - "path-expression-matcher": "^1.2.0", - "strnum": "^2.2.2" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -1222,388 +242,69 @@ "reusify": "^1.0.4" } }, - "node_modules/figures": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", - "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^1.0.5" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-east-asian-width": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", - "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/get-uri": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", - "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", - "license": "MIT", - "dependencies": { - "basic-ftp": "^5.0.2", - "data-uri-to-buffer": "^6.0.2", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globby": { - "version": "16.1.1", - "resolved": "https://registry.npmjs.org/globby/-/globby-16.1.1.tgz", - "integrity": "sha512-dW7vl+yiAJSp6aCekaVnVJxurRv7DCOLyXqEG3RYMYUg7AuJ2jCqPkZTA8ooqC2vtnkaMcV5WfFBMuEnTu1OQg==", - "license": "MIT", - "dependencies": { - "@sindresorhus/merge-streams": "^4.0.0", - "fast-glob": "^3.3.3", - "ignore": "^7.0.5", - "is-path-inside": "^4.0.0", - "slash": "^5.1.0", - "unicorn-magic": "^0.4.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" + "to-regex-range": "^5.0.1" }, "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "node": ">=8" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "node_modules/get-east-asian-width": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", + "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", "license": "MIT", "engines": { - "node": ">= 4" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", - "license": "MIT" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "license": "ISC" - }, - "node_modules/inquirer": { - "version": "8.2.7", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", - "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", - "license": "MIT", + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", "dependencies": { - "@inquirer/external-editor": "^1.0.0", - "ansi-escapes": "^4.2.1", - "chalk": "^4.1.1", - "cli-cursor": "^3.1.0", - "cli-width": "^3.0.0", - "figures": "^3.0.0", - "lodash": "^4.17.21", - "mute-stream": "0.0.8", - "ora": "^5.4.1", - "run-async": "^2.4.0", - "rxjs": "^7.5.5", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0", - "through": "^2.3.6", - "wrap-ansi": "^6.0.1" + "is-glob": "^4.0.1" }, "engines": { - "node": ">=12.0.0" + "node": ">= 6" } }, - "node_modules/inquirer/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "node_modules/globby": { + "version": "16.1.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-16.1.1.tgz", + "integrity": "sha512-dW7vl+yiAJSp6aCekaVnVJxurRv7DCOLyXqEG3RYMYUg7AuJ2jCqPkZTA8ooqC2vtnkaMcV5WfFBMuEnTu1OQg==", "license": "MIT", "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" + "@sindresorhus/merge-streams": "^4.0.0", + "fast-glob": "^3.3.3", + "ignore": "^7.0.5", + "is-path-inside": "^4.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.4.0" }, "engines": { - "node": ">=10" + "node": ">=20" }, "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/ip-address": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", - "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", "license": "MIT", "engines": { - "node": ">= 12" + "node": ">= 4" } }, "node_modules/is-alphabetical": { @@ -1640,21 +341,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/is-docker": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-4.0.0.tgz", - "integrity": "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA==", - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1664,15 +350,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -1695,15 +372,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/is-interactive": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", - "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -1725,55 +393,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-unicode-supported": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", - "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "license": "MIT" - }, - "node_modules/jest-diff": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", - "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", - "license": "MIT", - "dependencies": { - "@jest/diff-sequences": "30.0.1", - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "pretty-format": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-diff/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -1786,21 +405,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, "node_modules/jsonc-parser": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", @@ -1816,18 +420,6 @@ "node": ">=0.10.0" } }, - "node_modules/jszip": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", - "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", - "license": "(MIT OR GPL-3.0-or-later)", - "dependencies": { - "lie": "~3.3.0", - "pako": "~1.0.2", - "readable-stream": "~2.3.6", - "setimmediate": "^1.0.5" - } - }, "node_modules/katex": { "version": "0.16.44", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.44.tgz", @@ -1844,15 +436,6 @@ "katex": "cli.js" } }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "license": "MIT", - "dependencies": { - "immediate": "~3.0.5" - } - }, "node_modules/linkify-it": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", @@ -1862,53 +445,6 @@ "uc.micro": "^2.0.0" } }, - "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "license": "MIT" - }, - "node_modules/log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "license": "MIT", - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/log-symbols/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/lru-cache": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.0.tgz", - "integrity": "sha512-sr8xPKE25m6vJVcrdn6NxtC0fVfuPowbscLypegRgOm0yXSqr5JNHCAY3hnusdJ7HRBW04j6Ip4khvHU778DuQ==", - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/markdown-it": { "version": "14.1.1", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", @@ -2030,15 +566,6 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", @@ -2552,220 +1079,38 @@ "node_modules/micromark-util-types": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", - "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "license": "ISC" - }, - "node_modules/netmask": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", - "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "license": "MIT", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", - "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", - "license": "MIT", - "dependencies": { - "bl": "^4.1.0", - "chalk": "^4.1.0", - "cli-cursor": "^3.1.0", - "cli-spinners": "^2.5.0", - "is-interactive": "^1.0.0", - "is-unicode-supported": "^0.1.0", - "log-symbols": "^4.1.0", - "strip-ansi": "^6.0.0", - "wcwidth": "^1.0.1" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/pac-proxy-agent": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", - "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", - "license": "MIT", - "dependencies": { - "@tootallnate/quickjs-emscripten": "^0.23.0", - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "get-uri": "^6.0.1", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.6", - "pac-resolver": "^7.0.1", - "socks-proxy-agent": "^8.0.5" - }, - "engines": { - "node": ">= 14" - } + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" }, - "node_modules/pac-resolver": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", - "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "license": "MIT", "dependencies": { - "degenerator": "^5.0.0", - "netmask": "^2.0.2" + "braces": "^3.0.3", + "picomatch": "^2.3.1" }, "engines": { - "node": ">= 14" + "node": ">=8.6" } }, - "node_modules/package-manager-detector": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", - "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", - "license": "MIT" - }, - "node_modules/packageurl-js": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-2.0.1.tgz", - "integrity": "sha512-N5ixXjzTy4QDQH0Q9YFjqIWd6zH6936Djpl2m9QNFmDv5Fum8q8BjkpAcHNMzOFE0IwQrFhJWex3AN6kS0OSwg==", + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", - "license": "(MIT AND Zlib)" - }, "node_modules/parse-entities": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", @@ -2785,37 +1130,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/path-expression-matcher": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.2.1.tgz", - "integrity": "sha512-d7gQQmLvAKXKXE2GeP9apIGbMYKz88zWdsn/BN2HRWVQsDFdUY36WSLTY0Jvd4HWi7Fb30gQ62oAOzdgJA6fZw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/picomatch": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", @@ -2843,72 +1157,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/pretty-format": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", - "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.5", - "ansi-styles": "^5.2.0", - "react-is": "^18.3.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "license": "MIT" - }, - "node_modules/proxy-agent": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", - "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "^4.3.4", - "http-proxy-agent": "^7.0.1", - "https-proxy-agent": "^7.0.3", - "lru-cache": "^7.14.1", - "pac-proxy-agent": "^7.0.1", - "proxy-from-env": "^1.1.0", - "socks-proxy-agent": "^8.0.2" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-agent/node_modules/lru-cache": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", - "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, "node_modules/punycode.js": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", @@ -2938,73 +1186,6 @@ ], "license": "MIT" }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "license": "MIT" - }, - "node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/restore-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", - "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", - "license": "MIT", - "dependencies": { - "onetime": "^5.1.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -3015,15 +1196,6 @@ "node": ">=0.10.0" } }, - "node_modules/run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -3047,66 +1219,6 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/rxjs": { - "version": "7.8.2", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", - "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.1.0" - } - }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", - "license": "MIT" - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "license": "ISC" - }, - "node_modules/simple-git": { - "version": "3.33.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.33.0.tgz", - "integrity": "sha512-D4V/tGC2sjsoNhoMybKyGoE+v8A60hRawKQ1iFRA1zwuDgGZCBJ4ByOzZ5J8joBbi4Oam0qiPH+GhzmSBwbJng==", - "license": "MIT", - "dependencies": { - "@kwsites/file-exists": "^1.1.1", - "@kwsites/promise-deferred": "^1.1.1", - "debug": "^4.4.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/steveukx/git-js?sponsor=1" - } - }, "node_modules/slash": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", @@ -3119,16 +1231,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "license": "MIT", - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" - } - }, "node_modules/smol-toml": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", @@ -3141,162 +1243,6 @@ "url": "https://github.com/sponsors/cyyynthia" } }, - "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", - "license": "MIT", - "dependencies": { - "ip-address": "^10.0.1", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/strnum": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.2.tgz", - "integrity": "sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-hyperlinks": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", - "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/terminal-link": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", - "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==", - "license": "MIT", - "dependencies": { - "ansi-escapes": "^4.2.1", - "supports-hyperlinks": "^2.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "license": "MIT" - }, - "node_modules/tiny-async-pool": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-2.1.0.tgz", - "integrity": "sha512-ltAHPh/9k0STRQqaoUX52NH4ZQYAJz24ZAEwf1Zm+HYg3l9OXTWeqWKyYsHu40wF/F0rxd2N2bk5sLvX2qlSvg==", - "license": "MIT" - }, - "node_modules/tinyexec": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", - "integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -3309,33 +1255,6 @@ "node": ">=8.0" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/typanion": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/typanion/-/typanion-3.14.0.tgz", - "integrity": "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==", - "license": "MIT", - "workspaces": [ - "website" - ] - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", @@ -3353,58 +1272,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/upath": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", - "integrity": "sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==", - "license": "MIT", - "engines": { - "node": ">=4", - "yarn": "*" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/wcwidth": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", - "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", - "license": "MIT", - "dependencies": { - "defaults": "^1.0.3" - } - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } } } } diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json index a7ec2311e71d2..a40c7f7c3e2ae 100644 --- a/scripts/environment/npm-tools/package.json +++ b/scripts/environment/npm-tools/package.json @@ -2,7 +2,7 @@ "private": true, "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", "dependencies": { - "@datadog/datadog-ci": "5.12.0", + "@datadog/datadog-ci": "5.13.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } From 16458a84ba60660bbdc4258e3245981102da2921 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:48:09 -0400 Subject: [PATCH 077/364] chore(website deps): bump follow-redirects from 1.15.11 to 1.16.0 in /website (#25189) chore(website deps): bump follow-redirects in /website Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.11 to 1.16.0. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.11...v1.16.0) --- updated-dependencies: - dependency-name: follow-redirects dependency-version: 1.16.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 31254815a65f1..c2d1c9ed279dc 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2383,9 +2383,9 @@ fn.name@1.x.x: integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== follow-redirects@^1.15.11: - version "1.15.11" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.11.tgz#777d73d72a92f8ec4d2e410eb47352a56b8e8340" - integrity sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ== + version "1.16.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.16.0.tgz#28474a159d3b9d11ef62050a14ed60e4df6d61bc" + integrity sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw== form-data@^4.0.5: version "4.0.5" From e32661adb9138ff721d882f059f35eaf792d355c Mon Sep 17 00:00:00 2001 From: Peter Ehikhuemen Date: Tue, 14 Apr 2026 18:53:04 -0400 Subject: [PATCH 078/364] feat(aws_s3 sink): add Parquet encoder with schema_file and auto infer schema support (#25156) * feat(codecs): add Parquet encoder with schema_file and schema_mode support * fix broken tests * remove benches stuff * fix the integration tests * Add compression level to compression type * update teh aws_s3.cue file * Minor updates based on my own PR review of szibis initial PR * fix minor issue from batch encoder * fix tests from szibis PR * Fix the tests * Fix cargo fmt and deny issues * fix cargo markdown and license errors * Revert cargo.lock serde_json back to what it was * rebuild cargo.lock, don't want to pull in non needed upgrades * update warning and licenses * revert weird test updates * merge internal_events * cargo fmt fix * Throw a validation error if someone tries to use arrow encoding with amazon s3 sink * update licenses after re-installing dd-rust-license-tool * Short circuit when there's a serialization error instead of just emitting an error * PR comments feedback * emit dropped events metric when there's an error in encoding * put JsonSerializationError under parquet feature * emit ArrowWriterError and dropped events * remove formatting changes in cargo.toml * Apply suggestions from May's code review Co-authored-by: May Lee * remove whitespace updates from lib/codecs/cargo.toml --------- Co-authored-by: May Lee --- .github/actions/spelling/expect.txt | 1 + .gitignore | 7 + Cargo.lock | 64 ++ Cargo.toml | 10 +- LICENSE-3rdparty.csv | 5 + .../aws_s3_parquet_encoding.feature.md | 10 + deny.toml | 1 + lib/codecs/Cargo.toml | 11 +- lib/codecs/src/encoding/encoder.rs | 16 +- lib/codecs/src/encoding/format/arrow.rs | 62 +- lib/codecs/src/encoding/format/mod.rs | 6 + lib/codecs/src/encoding/format/parquet.rs | 943 ++++++++++++++++++ lib/codecs/src/encoding/mod.rs | 4 + lib/codecs/src/encoding/serializer.rs | 30 +- lib/codecs/src/internal_events.rs | 101 +- lib/vector-lib/Cargo.toml | 1 + src/sinks/aws_s3/config.rs | 334 ++++++- src/sinks/aws_s3/integration_tests.rs | 106 ++ src/sinks/aws_s3/sink.rs | 8 +- src/sinks/clickhouse/config.rs | 12 +- .../cue/reference/components/sinks/aws_s3.cue | 129 +++ 21 files changed, 1818 insertions(+), 43 deletions(-) create mode 100644 changelog.d/aws_s3_parquet_encoding.feature.md create mode 100644 lib/codecs/src/encoding/format/parquet.rs diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ddd945006f1c4..3656732c51bba 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -604,6 +604,7 @@ transitioning Trauring Treemap trialled +Trino Trivago trivy trl diff --git a/.gitignore b/.gitignore index 1963fcca25fcc..a713eeb297f9a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ cargo-timing*.html target tests/data/wasm/*/target bench_output.txt +rust-analyzer.toml # Python *.pyc @@ -64,3 +65,9 @@ volumes/ # LLM tools copilot-instructions.md + +# local files not checked in +local/ + +# vscode +.vscode/ diff --git a/Cargo.lock b/Cargo.lock index d013bc09432ec..ae4c753529775 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -382,6 +382,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", + "chrono-tz", "half", "hashbrown 0.16.0", "num", @@ -2428,6 +2429,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "chrono-tz", "csv-core", "derivative", "derive_more 2.1.1", @@ -2440,6 +2442,7 @@ dependencies = [ "metrics", "opentelemetry-proto", "ordered-float 4.6.0", + "parquet", "pin-project", "prost 0.12.6", "prost-reflect", @@ -5525,6 +5528,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "inventory" version = "0.3.22" @@ -7568,6 +7577,37 @@ dependencies = [ "windows-link 0.2.0", ] +[[package]] +name = "parquet" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +dependencies = [ + "ahash 0.8.11", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.16.0", + "lz4_flex", + "num", + "num-bigint", + "paste", + "seq-macro", + "snap", + "thrift", + "twox-hash", + "zstd 0.13.2", +] + [[package]] name = "parse-size" version = "1.1.0" @@ -7580,6 +7620,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13242a5ce97f39a8095d03c8b273e91d09f2690c0b7d69a2af844941115bab24" +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pastey" version = "0.2.1" @@ -9755,6 +9801,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -11021,6 +11073,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + [[package]] name = "tikv-jemalloc-sys" version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" @@ -12348,6 +12411,7 @@ dependencies = [ "openssl-probe", "openssl-src", "ordered-float 4.6.0", + "parquet", "pastey", "percent-encoding", "pin-project", diff --git a/Cargo.toml b/Cargo.toml index 84d89c50096f9..6b233580143c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,7 @@ tonic-build = { version = "0.11", default-features = false, features = ["transpo tonic-health = { version = "0.11", default-features = false } tonic-reflection = { version = "0.11", default-features = false, features = ["server"] } tracing = { version = "0.1.44", default-features = false } -tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } +tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } url = { version = "2.5.4", default-features = false, features = ["serde"] } uuid = { version = "1.22.0", features = ["v4", "v7", "serde", "fast-rng"] } vector-config = { path = "lib/vector-config" } @@ -353,6 +353,13 @@ async-compression = { version = "0.4.27", default-features = false, features = [ apache-avro = { version = "0.16.0", default-features = false, optional = true } arrow = { version = "56.2.0", default-features = false, features = ["ipc"], optional = true } arrow-schema = { version = "56.2.0", default-features = false, optional = true } +parquet = { version = "56.2.0", default-features = false, features = [ + "arrow", + "snap", + "zstd", + "lz4", + "flate2-rust_backened", +], optional = true } axum = { version = "0.6.20", default-features = false } base64 = { workspace = true, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } @@ -609,6 +616,7 @@ enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs codecs-arrow = ["dep:arrow", "dep:arrow-schema", "vector-lib/arrow"] +codecs-parquet = ["dep:parquet", "codecs-arrow", "vector-lib/parquet"] codecs-opentelemetry = ["vector-lib/opentelemetry"] codecs-syslog = ["vector-lib/syslog"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 9a135f0479f51..ac6f8bc2acb4a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -382,6 +382,7 @@ inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley , Joshka" +integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis @@ -533,8 +534,10 @@ pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras +parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow parse-size,https://github.com/kennytm/parse-size,MIT,kennytm passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel +paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald @@ -674,6 +677,7 @@ secrecy,https://github.com/iqlusioninc/crates/tree/main/secrecy,Apache-2.0 OR MI security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay +seq-macro,https://github.com/dtolnay/seq-macro,MIT OR Apache-2.0,David Tolnay serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy serde-toml-merge,https://github.com/jdrouet/serde-toml-merge,MIT,Jeremie Drouet @@ -756,6 +760,7 @@ terminal_size,https://github.com/eminence/terminal-size,MIT OR Apache-2.0,Andrew thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras +thrift,https://github.com/apache/thrift/tree/master/lib/rs,Apache-2.0,Apache Thrift Developers tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , The TiKV Project Developers" tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , Simon Sapin , Steven Fackler , The TiKV Project Developers" time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" diff --git a/changelog.d/aws_s3_parquet_encoding.feature.md b/changelog.d/aws_s3_parquet_encoding.feature.md new file mode 100644 index 0000000000000..afaa01cc70c26 --- /dev/null +++ b/changelog.d/aws_s3_parquet_encoding.feature.md @@ -0,0 +1,10 @@ +Add Apache Parquet batch encoding support for the `aws_s3` sink with flexible schema definitions. + +Events can now be encoded as Parquet columnar files with multiple schema input options: + +- **Native Parquet schema** — automatically generate a schema or supply `.schema` file +- **Configurable compression** - (Snappy, ZSTD, GZIP, LZ4, None). + +Enable the `codecs-parquet` feature and configure `batch_encoding` with `codec = "parquet"` in the S3 sink configuration. + +authors: szibis petere-datadog diff --git a/deny.toml b/deny.toml index b8e0be77f41c8..c19b0a23f5ee6 100644 --- a/deny.toml +++ b/deny.toml @@ -44,4 +44,5 @@ ignore = [ { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, ] diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 90a5fd2780109..b9043b7223d8a 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -15,9 +15,17 @@ path = "tests/bin/generate-avro-fixtures.rs" [dependencies] apache-avro = { version = "0.20.0", default-features = false } arrow = { version = "56.2.0", default-features = false, features = ["ipc", "json"], optional = true } +parquet = { version = "56.2.0", default-features = false, features = [ + "arrow", + "snap", + "zstd", + "lz4", + "flate2-rust_backened", +], optional = true } async-trait.workspace = true bytes.workspace = true chrono.workspace = true +chrono-tz.workspace = true rust_decimal.workspace = true csv-core = { version = "0.1.13", default-features = false } derivative.workspace = true @@ -69,7 +77,8 @@ uuid.workspace = true vrl.workspace = true [features] -arrow = ["dep:arrow"] +arrow = ["dep:arrow", "arrow/chrono-tz"] +parquet = ["dep:parquet", "arrow"] opentelemetry = ["dep:opentelemetry-proto"] syslog = ["dep:syslog_loose", "dep:strum", "dep:derive_more", "dep:serde-aux", "dep:toml"] test = [] diff --git a/lib/codecs/src/encoding/encoder.rs b/lib/codecs/src/encoding/encoder.rs index 4924dd05447b1..b3ca9a00216ec 100644 --- a/lib/codecs/src/encoding/encoder.rs +++ b/lib/codecs/src/encoding/encoder.rs @@ -5,6 +5,8 @@ use vector_core::event::Event; #[cfg(feature = "arrow")] use crate::encoding::ArrowStreamSerializer; +#[cfg(feature = "parquet")] +use crate::encoding::ParquetSerializer; use crate::{ encoding::{Error, Framer, Serializer}, internal_events::{EncoderFramingError, EncoderSerializeError}, @@ -16,6 +18,9 @@ pub enum BatchSerializer { /// Arrow IPC stream format serializer. #[cfg(feature = "arrow")] Arrow(ArrowStreamSerializer), + /// Parquet format serializer. + #[cfg(feature = "parquet")] + Parquet(Box), } /// An encoder that encodes batches of events. @@ -36,10 +41,12 @@ impl BatchEncoder { } /// Get the HTTP content type. - #[cfg(feature = "arrow")] + #[cfg(any(feature = "arrow", feature = "parquet"))] pub const fn content_type(&self) -> &'static str { match &self.serializer { BatchSerializer::Arrow(_) => "application/vnd.apache.arrow.stream", + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(_) => "application/vnd.apache.parquet", } } } @@ -63,6 +70,11 @@ impl tokio_util::codec::Encoder> for BatchEncoder { } }) } + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(serializer) => serializer + .encode(events, buffer) + .map_err(Error::SerializingError), + #[allow(unreachable_patterns)] _ => unreachable!("BatchSerializer cannot be constructed without encode()"), } } @@ -74,7 +86,7 @@ pub enum EncoderKind { /// Uses framing to encode individual events Framed(Box>), /// Encodes events in batches without framing - #[cfg(feature = "arrow")] + #[cfg(any(feature = "arrow", feature = "parquet"))] Batch(BatchEncoder), } diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index 236d6292d375e..b8c86239ddb6a 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -6,6 +6,7 @@ use arrow::{ datatypes::{DataType, Field, Fields, Schema, SchemaRef}, + error::ArrowError, ipc::writer::StreamWriter, json::reader::ReaderBuilder, record_batch::RecordBatch, @@ -203,7 +204,13 @@ pub fn encode_events_to_arrow_ipc_stream( return Err(ArrowEncodingError::NoEvents); } - let record_batch = build_record_batch(schema, events)?; + let json_values = vector_log_events_to_json_values(events).map_err(|e| { + ArrowEncodingError::RecordBatchCreation { + source: ArrowError::JsonError(e.to_string()), + } + })?; + + let record_batch = build_record_batch(schema, &json_values)?; let mut buffer = BytesMut::new().writer(); let mut writer = @@ -271,7 +278,7 @@ fn make_field_nullable(field: &Field) -> Result { /// Find non-nullable schema fields that are missing or null in any of the given events. pub fn find_null_non_nullable_fields<'a>( schema: &'a Schema, - values: &[&vrl::value::Value], + values: &[serde_json::Value], ) -> Vec<&'a str> { schema .fields() @@ -282,38 +289,41 @@ pub fn find_null_non_nullable_fields<'a>( value .as_object() .and_then(|map| map.get(field.name().as_str())) - .is_none_or(vrl::value::Value::is_null) + .is_none_or(serde_json::Value::is_null) }) }) .map(|field| field.name().as_str()) .collect() } -/// Build an Arrow RecordBatch from a slice of events using the provided schema. -fn build_record_batch( - schema: SchemaRef, +pub(crate) fn vector_log_events_to_json_values( events: &[Event], -) -> Result { - let values: Vec<_> = events +) -> Result, serde_json::Error> { + events .iter() .filter_map(Event::maybe_as_log) - .map(|log| log.value()) - .collect(); + .map(serde_json::to_value) + .collect() +} +/// Build an Arrow RecordBatch from a slice of events using the provided schema. +pub(crate) fn build_record_batch( + schema: SchemaRef, + values: &[serde_json::Value], +) -> Result { if values.is_empty() { return Err(ArrowEncodingError::NoEvents); } - let missing = find_null_non_nullable_fields(&schema, &values); + let missing = find_null_non_nullable_fields(&schema, values); if !missing.is_empty() { - for field_name in &missing { - let error: vector_common::Error = Box::new(ArrowEncodingError::NullConstraint { - field_name: field_name.to_string(), - }); - vector_common::internal_event::emit( - crate::internal_events::EncoderNullConstraintError { error: &error }, - ); - } + let error: vector_common::Error = Box::new(ArrowEncodingError::NullConstraint { + field_name: missing.join(", "), + }); + vector_common::internal_event::emit(crate::internal_events::EncoderNullConstraintError { + error: &error, + count: values.len(), + }); return Err(ArrowEncodingError::NullConstraint { field_name: missing.join(", "), }); @@ -323,7 +333,7 @@ fn build_record_batch( .build_decoder() .context(RecordBatchCreationSnafu)?; - decoder.serialize(&values).context(ArrowJsonDecodeSnafu)?; + decoder.serialize(values).context(ArrowJsonDecodeSnafu)?; decoder .flush() @@ -885,8 +895,10 @@ mod tests { ("a", Value::Bytes("val".into())), ("b", Value::Integer(42)), ]); - let value = event.as_log().value(); - let missing = find_null_non_nullable_fields(&schema, &[value]); + let missing = find_null_non_nullable_fields( + &schema, + &vector_log_events_to_json_values(&[event]).unwrap(), + ); assert!( missing.is_empty(), "Expected no missing fields, got: {missing:?}" @@ -898,8 +910,10 @@ mod tests { let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]); let event = create_event(vec![("a", Value::Null)]); - let value = event.as_log().value(); - let missing = find_null_non_nullable_fields(&schema, &[value]); + let missing = find_null_non_nullable_fields( + &schema, + &vector_log_events_to_json_values(&[event]).unwrap(), + ); assert_eq!(missing, vec!["a"]); } } diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index 85bc094b26947..f760ea5c8dd83 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -16,6 +16,8 @@ mod native; mod native_json; #[cfg(feature = "opentelemetry")] mod otlp; +#[cfg(feature = "parquet")] +mod parquet; mod protobuf; mod raw_message; #[cfg(feature = "syslog")] @@ -24,6 +26,10 @@ mod text; use std::fmt::Debug; +#[cfg(feature = "parquet")] +pub use self::parquet::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializer, ParquetSerializerConfig, +}; #[cfg(feature = "arrow")] pub use arrow::{ ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs new file mode 100644 index 0000000000000..7349119371a5b --- /dev/null +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -0,0 +1,943 @@ +//! Parquet batch format codec for batched event encoding +//! +//! Provides Apache Parquet format encoding with schema file support and auto-inference. +//! Reuses the Arrow record batch building logic from the Arrow IPC codec, +//! then writes the batch as a complete Parquet file using `ArrowWriter`. + +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::error::ArrowError; +use arrow::json::reader::infer_json_schema_from_iterator; +use arrow::record_batch::RecordBatch; +use bytes::{BufMut, BytesMut}; +use derivative::Derivative; +use parquet::arrow::ArrowWriter; +use parquet::basic::ZstdLevel; +use parquet::basic::{Compression as ParquetCodecCompression, GzipLevel}; +use parquet::file::properties::WriterProperties; +use std::io::{Error, ErrorKind}; +use tracing::warn; +use vector_common::internal_event::{ + ComponentEventsDropped, Count, InternalEventHandle, Registered, UNINTENTIONAL, emit, register, +}; +use vector_config::configurable_component; +use vector_core::event::Event; + +use super::arrow::{ArrowEncodingError, build_record_batch}; +use crate::encoding::format::arrow::vector_log_events_to_json_values; +use crate::internal_events::{ArrowWriterError, JsonSerializationError, SchemaGenerationError}; + +type EventsDroppedError = ComponentEventsDropped<'static, UNINTENTIONAL>; + +/// Compression algorithm and optional level for archive objects. +#[configurable_component] +#[derive(Default, Copy, Clone, Debug, PartialEq)] +#[serde(tag = "algorithm", rename_all = "snake_case")] +pub enum ParquetCompression { + /// Zstd compression. Level must be between 1 and 21. + Zstd { + /// Compression level (1–21). This is the range Vector currently supports; higher values compress more but are slower. + #[configurable(validation(range(min = 1, max = 21)))] + level: u8, + }, + /// Gzip compression. Level must be between 1 and 9. + Gzip { + /// Compression level (1–9). This is the range Vector currently supports; higher values compress more but are slower. + #[configurable(validation(range(min = 1, max = 9)))] + level: u8, + }, + + /// Snappy compression (no level). + #[default] + Snappy, + + /// LZ4 raw compression + Lz4, + + /// No compression + None, +} + +impl TryFrom for ParquetCodecCompression { + type Error = parquet::errors::ParquetError; + fn try_from( + value: ParquetCompression, + ) -> Result { + match value { + ParquetCompression::None => Ok(ParquetCodecCompression::UNCOMPRESSED), + ParquetCompression::Snappy => Ok(ParquetCodecCompression::SNAPPY), + ParquetCompression::Zstd { level } => Ok(ParquetCodecCompression::ZSTD( + ZstdLevel::try_new(level.into())?, + )), + ParquetCompression::Gzip { level } => Ok(ParquetCodecCompression::GZIP( + GzipLevel::try_new(level.into())?, + )), + ParquetCompression::Lz4 => Ok(ParquetCodecCompression::LZ4_RAW), + } + } +} + +/// Schema handling mode. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ParquetSchemaMode { + /// Missing fields become null. Extra fields are silently dropped. + #[default] + Relaxed, + /// Missing fields become null. Extra fields cause an error. + Strict, + /// Auto infer schema based on the batch. No schema file needed. + AutoInfer, +} + +/// Configuration for the Parquet serializer. +/// +/// Encodes events as Apache Parquet columnar files, optimized for analytical queries +/// via Athena, Trino, Spark, and other columnar query engines. +/// +/// Either `schema_file` must be provided, or `schema_mode` must be set to `auto_infer`. +#[configurable_component] +#[derive(Clone, Debug, Default)] +pub struct ParquetSerializerConfig { + /// Path to a native Parquet schema file (`.schema`). + /// + /// Required unless `schema_mode` is `auto_infer`. The file must contain a valid + /// Parquet message type definition. + #[serde(default)] + pub schema_file: Option, + + /// Compression codec applied per column page inside the Parquet file. + #[serde(default)] + #[configurable(derived)] + pub compression: ParquetCompression, + + /// Controls how events with fields not present in the schema are handled. + #[serde(default)] + #[configurable(derived)] + pub schema_mode: ParquetSchemaMode, +} + +impl ParquetSerializerConfig { + /// Resolve the Arrow schema from the configured schema source. + fn resolve_schema(&self) -> Result> { + if self.schema_mode == ParquetSchemaMode::AutoInfer { + return Ok(Schema::empty()); + } + + let path = self + .schema_file + .as_ref() + .ok_or("schema_file is required unless schema_mode is auto_infer")?; + + let content = read_schema_file(path, "schema_file")?; + let parquet_type = parquet::schema::parser::parse_message_type(&content) + .map_err(|e| format!("Failed to parse Parquet schema: {e}"))?; + let schema_desc = parquet::schema::types::SchemaDescriptor::new(Arc::new(parquet_type)); + let arrow_schema = parquet::arrow::parquet_to_arrow_schema(&schema_desc, None) + .map_err(|e| format!("Failed to convert Parquet schema to Arrow: {e}"))?; + Ok(arrow_schema) + } + + /// The data type of events that are accepted by `ParquetSerializer`. + pub fn input_type(&self) -> vector_core::config::DataType { + vector_core::config::DataType::Log + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> vector_core::schema::Requirement { + vector_core::schema::Requirement::empty() + } +} + +fn read_schema_file( + path: &std::path::Path, + field_name: &str, +) -> Result> { + const MAX_SCHEMA_FILE_SIZE: u64 = 10 * 1024 * 1024; // 10 MB + let display = path.display(); + let metadata = std::fs::metadata(path) + .map_err(|e| format!("Failed to read {field_name} '{display}': {e}"))?; + if metadata.len() > MAX_SCHEMA_FILE_SIZE { + return Err(format!( + "{field_name} '{display}' is too large ({} bytes, max {MAX_SCHEMA_FILE_SIZE})", + metadata.len() + ) + .into()); + } + std::fs::read_to_string(path) + .map_err(|e| format!("Failed to read {field_name} '{display}': {e}").into()) +} + +/// Check the resolved Arrow schema for data types unsupported by the JSON-based +/// encode path (`arrow::json::reader::ReaderBuilder`). Binary variants are +/// accepted by Parquet/Arrow at the schema level but the JSON decoder rejects +/// them at runtime, so we fail fast here at config time. +fn reject_unsupported_arrow_types( + schema: &Schema, +) -> Result<(), Box> { + fn check_field(field: &Field, path: &str, bad: &mut Vec) { + let name = if path.is_empty() { + field.name().to_string() + } else { + format!("{path}.{}", field.name()) + }; + match field.data_type() { + DataType::Binary | DataType::LargeBinary | DataType::FixedSizeBinary(_) => { + bad.push(format!("'{name}' ({:?})", field.data_type())); + } + DataType::Struct(fields) => { + for f in fields { + check_field(f, &name, bad); + } + } + DataType::List(inner) | DataType::LargeList(inner) => { + check_field(inner, &name, bad); + } + DataType::Map(entries_field, _) => { + if let DataType::Struct(kv) = entries_field.data_type() { + for f in kv { + check_field(f, &name, bad); + } + } + } + _ => {} + } + } + + let mut bad = Vec::new(); + for field in schema.fields() { + check_field(field, "", &mut bad); + } + if !bad.is_empty() { + return Err(format!( + "Schema contains binary field(s) unsupported by the JSON-based Arrow encoder: {}. \ + Use Utf8 for base64/hex-encoded data instead.", + bad.join(", ") + ) + .into()); + } + Ok(()) +} + +/// Parquet batch serializer. +#[derive(Derivative)] +#[derivative(Debug, Clone)] +pub struct ParquetSerializer { + schema: SchemaRef, + writer_props: Arc, + schema_mode: ParquetSchemaMode, + /// Pre-built set of schema field names for O(1) strict-mode lookups. + schema_field_names: HashSet, + + #[derivative(Debug = "ignore")] + events_dropped_handle: Registered, +} + +impl ParquetSerializer { + /// Create a new `ParquetSerializer` from the given configuration. + pub fn new( + config: ParquetSerializerConfig, + ) -> Result> { + let schema = config.resolve_schema()?; + reject_unsupported_arrow_types(&schema)?; + let schema_ref = SchemaRef::new(schema); + + let schema_field_names = schema_ref + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(); + + let writer_props = Arc::new( + WriterProperties::builder() + .set_compression(config.compression.try_into()?) + .build(), + ); + + Ok(Self { + schema: schema_ref, + writer_props, + schema_mode: config.schema_mode, + schema_field_names, + events_dropped_handle: register(EventsDroppedError::from( + "Events could not be serialized to parquet", + )), + }) + } + + /// Returns the MIME content type for Parquet data. + pub const fn content_type(&self) -> &'static str { + "application/vnd.apache.parquet" + } + + /// Writes `record_batch` into `buffer` as a complete Parquet file. + /// + /// On failure, emits an [`ArrowWriterError`] internal event (which also + /// increments `component_errors_total` and emits the events-dropped metric) + /// before returning the error. + fn write_record_batch( + &self, + record_batch: &RecordBatch, + buffer: &mut BytesMut, + event_count: usize, + ) -> Result<(), parquet::errors::ParquetError> { + let mut writer = ArrowWriter::try_new( + buffer.writer(), + Arc::clone(record_batch.schema_ref()), + Some((*self.writer_props).clone()), + ) + .inspect_err(|e| { + emit(ArrowWriterError { + error: e, + batch_count: event_count, + }); + })?; + + writer.write(record_batch).inspect_err(|e| { + emit(ArrowWriterError { + error: e, + batch_count: event_count, + }); + })?; + + writer.close().inspect_err(|e| { + emit(ArrowWriterError { + error: e, + batch_count: event_count, + }); + })?; + + Ok(()) + } +} + +impl tokio_util::codec::Encoder> for ParquetSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, events: Vec, buffer: &mut BytesMut) -> Result<(), Self::Error> { + if events.is_empty() { + return Ok(()); + } + + let json_values = match vector_log_events_to_json_values(&events) { + Ok(values) => values, + Err(e) => { + emit(JsonSerializationError { + error: &e, + batch_count: events.len(), + }); + return Err(Box::new(e)); + } + }; + + let non_log_count = events.len() - json_values.len(); + + if non_log_count > 0 { + warn!( + message = "Non-log events dropped by Parquet encoder ", + %non_log_count, + internal_log_rate_secs = 10, + ); + self.events_dropped_handle.emit(Count(non_log_count)) + } + + if json_values.is_empty() { + return Ok(()); + } + + match self.schema_mode { + // In strict mode, check for extra top-level fields not in the schema. + ParquetSchemaMode::Strict => { + for event in &events { + if let Some(log) = event.maybe_as_log() + && let Some(object_map) = log.as_map() + { + for top_level in object_map.keys() { + if !self.schema_field_names.contains(top_level.as_str()) { + self.events_dropped_handle.emit(Count(events.len())); + return Err(Box::new(ArrowEncodingError::SchemaFetchError { + message: format!( + "Strict schema mode: event contains field '{top_level}' not in schema", + ), + })); + } + } + } + } + } + ParquetSchemaMode::AutoInfer => { + let schema = ParquetSchemaGenerator::infer_schema(&json_values)?; + self.schema = Arc::new(ParquetSchemaGenerator::try_normalize_schema( + &events, schema, + )); + } + ParquetSchemaMode::Relaxed => {} + } + + let record_batch = + build_record_batch(Arc::clone(&self.schema), &json_values).map_err(Box::new)?; + + self.write_record_batch(&record_batch, buffer, json_values.len()) + .map_err(Box::new)?; + + Ok(()) + } +} + +pub struct ParquetSchemaGenerator {} + +impl ParquetSchemaGenerator { + pub fn infer_schema(events: &[serde_json::Value]) -> Result { + let schema = infer_json_schema_from_iterator(events.iter().map(Ok::<_, ArrowError>)) + .map_err(|e| { + emit(SchemaGenerationError { + error: &e, + batch_count: events.len(), + }); + Error::new(ErrorKind::InvalidData, e.to_string()) + })?; + + Ok(schema) + } + + /// Attempt to modify schema to set timestamp fields as Timestamp instead of Utf8. + /// Only works for top-level fields. + fn try_normalize_schema(events: &[Event], schema: Schema) -> Schema { + let mut ts_seen: HashSet = HashSet::new(); + let mut non_ts_seen: HashSet = HashSet::new(); + + for event in events.iter().filter_map(Event::maybe_as_log) { + if let Some(object_map) = event.as_map() { + for (path, value) in object_map { + if value.is_timestamp() { + ts_seen.insert(path.to_string()); + } else if !value.is_null() { + non_ts_seen.insert(path.to_string()); + } + } + } + } + + let new_fields: Vec = schema + .fields() + .iter() + .map(|f| { + if ts_seen.contains(f.name()) && !non_ts_seen.contains(f.name()) { + Field::new( + f.name(), + DataType::Timestamp( + arrow::datatypes::TimeUnit::Microsecond, + Some("UTC".into()), + ), + f.is_nullable(), + ) + } else { + f.as_ref().clone() + } + }) + .collect(); + + Schema::new_with_metadata(new_fields, schema.metadata().clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::reader::RowIter; + use tokio_util::codec::Encoder; + use vector_core::event::LogEvent; + + fn create_event(fields: Vec<(&str, V)>) -> Event + where + V: Into, + { + let mut log = LogEvent::default(); + for (key, value) in fields { + log.insert(key, value.into()); + } + Event::Log(log) + } + + fn assert_parquet_magic(data: &[u8]) { + assert!(data.len() >= 4, "Output too short to be valid Parquet"); + assert_eq!(&data[..4], b"PAR1", "Missing Parquet magic bytes"); + } + + fn parquet_row_count(data: &[u8]) -> usize { + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(data)).expect("Invalid Parquet file"); + let iter = RowIter::from_file_into(Box::new(reader)); + iter.count() + } + + fn parquet_column_names(data: &[u8]) -> Vec { + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(data)).expect("Invalid Parquet file"); + let schema = reader.metadata().file_metadata().schema_descr(); + schema + .columns() + .iter() + .map(|c| c.name().to_string()) + .collect() + } + + fn parse_timestamp(s: &str) -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339(s) + .expect("invalid test timestamp") + .with_timezone(&chrono::Utc) + } + + fn demo_log_event( + message: &str, + timestamp: chrono::DateTime, + status_code: i64, + response_time_secs: f64, + ) -> Event { + use vector_core::event::Value; + let mut log = LogEvent::default(); + log.insert("host", "localhost"); + log.insert("message", message); + log.insert("service", "vector"); + log.insert("source_type", "demo_logs"); + log.insert("timestamp", Value::Timestamp(timestamp)); + log.insert("random_time", Value::Timestamp(timestamp)); + log.insert("status_code", Value::Integer(status_code)); + log.insert("response_time_secs", response_time_secs); + Event::Log(log) + } + + fn sample_events() -> Vec { + const EVENTS: [(&str, &str, i64, f64); 5] = [ + ( + "GET /api/v1/health HTTP/1.1", + "2026-03-05T20:49:08.037194Z", + 200, + 0.037, + ), + ( + "POST /api/v1/ingest HTTP/1.1", + "2026-03-05T20:49:09.038051Z", + 201, + 0.013, + ), + ( + "GET /metrics HTTP/1.1", + "2026-03-05T20:49:10.036612Z", + 200, + 0.022, + ), + ( + "DELETE /api/v1/resource HTTP/1.1", + "2026-03-05T20:49:11.537131Z", + 404, + 0.005, + ), + ( + "PATCH /api/v1/config HTTP/1.1", + "2026-03-05T20:49:12.037491Z", + 500, + 0.091, + ), + ]; + EVENTS + .iter() + .map(|(msg, ts, status, rt)| demo_log_event(msg, parse_timestamp(ts), *status, *rt)) + .collect() + } + + fn encode_autoinfer_and_read_schema( + events: Vec, + ) -> (arrow::datatypes::SchemaRef, usize) { + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should be created without a static schema"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("encoding should succeed"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + + let builder = ParquetRecordBatchReaderBuilder::try_new(data) + .expect("should build ParquetRecordBatchReaderBuilder"); + let schema = builder.schema().clone(); + let num_rows: usize = builder + .build() + .expect("should build reader") + .map(|b| b.expect("batch read error").num_rows()) + .sum(); + (schema, num_rows) + } + + /// Write a temporary Parquet schema file and return its path. + /// + /// `name` must be unique per test to avoid parallel-test races on the same file. + fn write_temp_schema(name: &str, content: &str) -> std::path::PathBuf { + use std::io::Write; + let path = std::env::temp_dir().join(format!( + "vector_parquet_test_{}_{}.schema", + std::process::id(), + name, + )); + let mut f = std::fs::File::create(&path).expect("Failed to create schema file"); + write!(f, "{content}").expect("Failed to write schema"); + path + } + + // ── AutoInfer mode ─────────────────────────────────────────────────────── + + #[test] + fn encode_input_produces_parquet_output() { + let events = sample_events(); + let n_events = events.len(); + let (schema, num_rows) = encode_autoinfer_and_read_schema(events); + + assert_eq!(num_rows, n_events, "row count should match event count"); + + for field_name in &["timestamp", "random_time"] { + let field = schema + .field_with_name(field_name) + .unwrap_or_else(|_| panic!("field '{field_name}' should exist in schema")); + assert!( + matches!( + field.data_type(), + DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, Some(tz)) if tz.as_ref() == "UTC" + ), + "'{field_name}' should be Timestamp(Microsecond, UTC), got {:?}", + field.data_type() + ); + } + + let status_field = schema + .field_with_name("status_code") + .expect("status_code field should exist"); + assert_eq!(status_field.data_type(), &DataType::Int64); + + let rt_field = schema + .field_with_name("response_time_secs") + .expect("response_time_secs field should exist"); + assert_eq!(rt_field.data_type(), &DataType::Float64); + + for field_name in &["host", "message", "service", "source_type"] { + let field = schema + .field_with_name(field_name) + .unwrap_or_else(|_| panic!("field '{field_name}' should exist in schema")); + assert_eq!(field.data_type(), &DataType::Utf8); + } + } + + #[test] + fn test_parquet_empty_events() { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let events: Vec = vec![]; + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Empty events should succeed"); + + assert!(buffer.is_empty(), "Buffer should be empty for empty events"); + } + + #[test] + fn test_parquet_compression_variants() { + let events = vec![create_event(vec![("msg", "hello world")])]; + + let compressions = vec![ + ParquetCompression::None, + ParquetCompression::Snappy, + ParquetCompression::Zstd { level: 1 }, + ParquetCompression::Gzip { level: 1 }, + ParquetCompression::Lz4, + ]; + + for compression in compressions { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression, + ..Default::default() + }) + .expect("Failed to create serializer"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events.clone(), &mut buffer) + .unwrap_or_else(|e| panic!("Encoding with {:?} failed: {}", compression, e)); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!( + parquet_row_count(&data), + 1, + "Wrong row count for {:?}", + compression + ); + } + } + + #[test] + fn test_parquet_output_has_footer() { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let events = vec![create_event(vec![("msg", "test")])]; + let mut buffer = BytesMut::new(); + serializer.encode(events, &mut buffer).unwrap(); + + let data = buffer.freeze(); + let len = data.len(); + assert!(len >= 8, "Parquet output too short"); + assert_eq!( + &data[len - 4..], + b"PAR1", + "Parquet footer magic bytes missing" + ); + } + + #[test] + fn test_writer_props_arc_shared() { + let serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + let cloned = serializer.clone(); + + assert_eq!(Arc::strong_count(&serializer.writer_props), 2); + drop(cloned); + assert_eq!(Arc::strong_count(&serializer.writer_props), 1); + } + + #[test] + fn test_mixed_log_and_non_log_events() { + use vector_core::event::{Metric, MetricKind, MetricValue}; + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let metric = Metric::new( + "cpu.usage", + MetricKind::Absolute, + MetricValue::Gauge { value: 42.0 }, + ); + let events = vec![ + create_event(vec![("msg", "hello")]), + Event::Metric(metric), + create_event(vec![("msg", "world")]), + ]; + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Mixed batch should succeed (non-log events dropped)"); + + assert_parquet_magic(&buffer); + assert_eq!(parquet_row_count(&buffer), 2); + } + + // ── Schema file mode ───────────────────────────────────────────────────── + + #[test] + fn test_parquet_schema_file() { + let schema_path = write_temp_schema( + "schema_file", + "message logs {\n required binary name (STRING);\n optional int64 age;\n}", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let mut serializer = + ParquetSerializer::new(config).expect("Should create serializer from schema file"); + + let mut log = LogEvent::default(); + log.insert("name", "alice"); + + let mut buffer = BytesMut::new(); + serializer + .encode(vec![Event::Log(log)], &mut buffer) + .expect("Encoding with schema file should succeed"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!(parquet_row_count(&data), 1); + + let columns = parquet_column_names(&data); + assert_eq!(columns, vec!["name", "age"]); + } + + #[test] + fn test_parquet_schema_file_not_found_error() { + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": "/nonexistent/path/schema.parquet" + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!(result.is_err(), "Missing schema file should error"); + assert!( + result.unwrap_err().to_string().contains("Failed to read"), + "Error should mention file read failure" + ); + } + + #[test] + fn test_parquet_schema_file_invalid_syntax_error() { + let schema_path = write_temp_schema( + "invalid_syntax", + "this is not valid parquet schema syntax !!!", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!(result.is_err(), "Invalid Parquet schema should error"); + assert!( + result + .unwrap_err() + .to_string() + .contains("Failed to parse Parquet schema"), + "Error should mention parsing failure" + ); + } + + #[test] + fn test_parquet_no_schema_error() { + let config = ParquetSerializerConfig::default(); + let result = ParquetSerializer::new(config); + assert!( + result.is_err(), + "Should fail without schema_file or auto_infer" + ); + } + + // ── Schema mode: strict / relaxed ──────────────────────────────────────── + + #[test] + fn test_parquet_strict_mode_rejects_extra_fields() { + let schema_path = write_temp_schema( + "strict_rejects", + "message logs {\n required binary name (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Strict, + ..Default::default() + }) + .expect("Failed to create strict serializer"); + + let events = vec![create_event(vec![("name", "alice"), ("city", "paris")])]; + let mut buffer = BytesMut::new(); + let result = serializer.encode(events, &mut buffer); + assert!(result.is_err(), "Strict mode should reject extra fields"); + assert!(result.unwrap_err().to_string().contains("city")); + } + + #[test] + fn test_parquet_strict_mode_allows_schema_fields() { + let schema_path = write_temp_schema( + "strict_allows", + "message logs {\n required binary name (STRING);\n required binary level (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Strict, + ..Default::default() + }) + .expect("Failed to create strict serializer"); + + let mut log = LogEvent::default(); + log.insert("name", "test"); + log.insert("level", "info"); + + let mut buffer = BytesMut::new(); + assert!( + serializer + .encode(vec![Event::Log(log)], &mut buffer) + .is_ok(), + "Strict mode should pass when all fields match schema" + ); + } + + #[test] + fn test_parquet_relaxed_mode_drops_extra_fields() { + let schema_path = write_temp_schema( + "relaxed_drops", + "message logs {\n required binary name (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Relaxed, + ..Default::default() + }) + .expect("Failed to create relaxed serializer"); + + let events = vec![create_event(vec![("name", "alice"), ("city", "paris")])]; + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Relaxed mode should drop extra fields silently"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!(parquet_row_count(&data), 1); + let columns = parquet_column_names(&data); + assert_eq!(columns, vec!["name"]); + } + + #[test] + fn test_parquet_schema_file_binary_without_string_annotation_rejected() { + // Native Parquet "binary" without (STRING) annotation resolves to Arrow Binary, + // which is rejected at config time. + let schema_path = write_temp_schema( + "binary_rejected", + "message logs {\n required binary name (STRING);\n optional binary raw_data;\n}", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!( + result.is_err(), + "Parquet binary without STRING annotation should be rejected" + ); + assert!( + result.unwrap_err().to_string().contains("raw_data"), + "Error should name the offending field" + ); + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 25f88406fbae9..a9f698f79153d 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -27,6 +27,10 @@ pub use format::{ }; #[cfg(feature = "opentelemetry")] pub use format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "parquet")] +pub use format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializer, ParquetSerializerConfig, +}; #[cfg(feature = "syslog")] pub use format::{SyslogSerializer, SyslogSerializerConfig}; pub use framing::{ diff --git a/lib/codecs/src/encoding/serializer.rs b/lib/codecs/src/encoding/serializer.rs index 536f836ad8163..c6081e3e2f274 100644 --- a/lib/codecs/src/encoding/serializer.rs +++ b/lib/codecs/src/encoding/serializer.rs @@ -8,6 +8,8 @@ use vector_core::{config::DataType, event::Event, schema}; use super::format::{ArrowStreamSerializer, ArrowStreamSerializerConfig}; #[cfg(feature = "opentelemetry")] use super::format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "parquet")] +use super::format::{ParquetSerializer, ParquetSerializerConfig}; #[cfg(feature = "syslog")] use super::format::{SyslogSerializer, SyslogSerializerConfig}; use super::{ @@ -160,17 +162,29 @@ pub enum BatchSerializerConfig { #[cfg(feature = "arrow")] #[serde(rename = "arrow_stream")] ArrowStream(ArrowStreamSerializerConfig), + /// Encodes events in [Apache Parquet][apache_parquet] columnar format. + /// + /// [apache_parquet]: https://parquet.apache.org/ + #[cfg(feature = "parquet")] + #[serde(rename = "parquet")] + Parquet(ParquetSerializerConfig), } -#[cfg(feature = "arrow")] +#[cfg(any(feature = "arrow", feature = "parquet"))] impl BatchSerializerConfig { - /// Build the `ArrowStreamSerializer` from this configuration. - pub fn build( + /// Build the batch serializer from this configuration. + pub fn build_batch_serializer( &self, - ) -> Result> { + ) -> Result> { match self { BatchSerializerConfig::ArrowStream(arrow_config) => { - Ok(ArrowStreamSerializer::new(arrow_config.clone())?) + let serializer = ArrowStreamSerializer::new(arrow_config.clone())?; + Ok(super::BatchSerializer::Arrow(serializer)) + } + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => { + let serializer = ParquetSerializer::new(parquet_config.clone())?; + Ok(super::BatchSerializer::Parquet(Box::new(serializer))) } } } @@ -178,14 +192,20 @@ impl BatchSerializerConfig { /// The data type of events that are accepted by this batch serializer. pub fn input_type(&self) -> DataType { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.input_type(), + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => parquet_config.input_type(), } } /// The schema required by the batch serializer. pub fn schema_requirement(&self) -> schema::Requirement { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.schema_requirement(), + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => parquet_config.schema_requirement(), } } } diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index 134fee16ecf8b..0ed24719dd8bc 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -157,6 +157,8 @@ impl InternalEvent for EncoderWriteError<'_, E> { pub struct EncoderNullConstraintError<'a> { /// The schema constraint error that occurred. pub error: &'a vector_common::Error, + /// The number of events dropped due to the constraint violation. + pub count: usize, } #[cfg(feature = "arrow")] @@ -178,7 +180,104 @@ impl InternalEvent for EncoderNullConstraintError<'_> { ) .increment(1); emit(ComponentEventsDropped:: { - count: 1, + count: self.count, + reason: CONSTRAINT_REASON, + }); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct SchemaGenerationError<'a> { + pub error: &'a arrow::error::ArrowError, + pub batch_count: usize, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for SchemaGenerationError<'_> { + fn emit(self) { + const REASON: &str = "Could not generate schema for batched events"; + error!( + message = REASON, + error = %self.error, + error_code = "parquet_schema_generation_failed", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = false, + ); + counter!( + "component_errors_total", + "error_code" => "parquet_schema_generation_failed", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: self.batch_count, + reason: REASON, + }); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct ArrowWriterError<'a> { + pub error: &'a parquet::errors::ParquetError, + pub batch_count: usize, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for ArrowWriterError<'_> { + fn emit(self) { + const REASON: &str = "Failed to write record batch with ArrowWriter."; + error!( + message = REASON, + error = %self.error, + error_code = "parquet_arrow_writer_failed", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = false, + ); + counter!( + "component_errors_total", + "error_code" => "parquet_arrow_writer_failed", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: self.batch_count, + reason: REASON, + }); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct JsonSerializationError<'a> { + pub error: &'a serde_json::Error, + pub batch_count: usize, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for JsonSerializationError<'_> { + fn emit(self) { + const CONSTRAINT_REASON: &str = "Could not serialize event to JSON."; + error!( + message = CONSTRAINT_REASON, + error = %self.error, + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: self.batch_count, reason: CONSTRAINT_REASON, }); } diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index 6539c82dfaa80..6f42af4dbb627 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -27,6 +27,7 @@ vrl = { workspace = true, optional = true } allocation-tracing = ["vector-top?/allocation-tracing"] api-client = ["dep:vector-api-client"] arrow = ["codecs/arrow"] +parquet = ["codecs/parquet"] api = ["vector-tap/api"] file-source = ["dep:file-source", "dep:file-source-common"] lua = ["vector-core/lua"] diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 08ba92245d65c..61b82105a1211 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -1,9 +1,13 @@ use aws_sdk_s3::Client as S3Client; use tower::ServiceBuilder; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::BatchEncoder; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ TimeZone, codecs::{ - TextSerializerConfig, + EncoderKind, TextSerializerConfig, encoding::{Framer, FramingConfig}, }, configurable::configurable_component, @@ -105,6 +109,16 @@ pub struct S3SinkConfig { #[serde(flatten)] pub encoding: EncodingConfigWithFraming, + /// Batch encoding configuration for columnar formats. + /// + /// When set, events are encoded together as a batch in a columnar format (e.g., Parquet) + /// instead of the standard per-event framing-based encoding. The columnar format handles + /// its own internal compression, so the top-level `compression` setting is bypassed. + #[cfg(feature = "codecs-parquet")] + #[configurable(derived)] + #[serde(default)] + pub batch_encoding: Option, + /// Compression configuration. /// /// All compression algorithms use the default compression level unless otherwise specified. @@ -176,6 +190,8 @@ impl GenerateConfig for S3SinkConfig { options: S3Options::default(), region: RegionOrEndpoint::default(), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::gzip_default(), batch: BatchConfig::default(), request: TowerRequestConfig::default(), @@ -201,6 +217,10 @@ impl SinkConfig for S3SinkConfig { } fn input(&self) -> Input { + #[cfg(feature = "codecs-parquet")] + if let Some(batch_config) = &self.batch_encoding { + return Input::new(batch_config.input_type()); + } Input::new(self.encoding.config().1.input_type()) } @@ -246,8 +266,62 @@ impl S3SinkConfig { let partitioner = S3KeyPartitioner::new(key_prefix, ssekms_key_id, None); let transformer = self.encoding.transformer(); + + // When batch_encoding is configured (e.g., Parquet), use batch mode + // with internal compression and appropriate file extension. + #[cfg(feature = "codecs-parquet")] + if let Some(batch_config) = &self.batch_encoding { + if !matches!(batch_config, BatchSerializerConfig::Parquet(_)) { + return Err( + "batch_encoding only supports encoding with parquet format for amazon s3 sink" + .into(), + ); + } + + let batch_serializer = batch_config.build_batch_serializer()?; + let batch_encoder = BatchEncoder::new(batch_serializer); + + // Auto-detect Content-Type from batch format. Users can still + // override via `options.content_type`; we only set it when unset. + let mut api_options = self.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = Some(batch_encoder.content_type().to_string()); + } + + let encoder = EncoderKind::Batch(batch_encoder); + + // Auto-detect file extension from batch format + let filename_extension = + self.filename_extension + .clone() + .or_else(|| match batch_config { + BatchSerializerConfig::Parquet(_) => Some("parquet".to_string()), + #[allow(unreachable_patterns)] + _ => None, + }); + + if self.compression != Compression::None { + warn!("Top level compression setting ignored when batch_encoding set to parquet.") + } + + let request_options = S3RequestOptions { + bucket: self.bucket.clone(), + api_options, + filename_extension, + filename_time_format: self.filename_time_format.clone(), + filename_append_uuid: self.filename_append_uuid, + encoder: (transformer, encoder), + // Batch formats handle their own compression internally + compression: Compression::None, + filename_tz_offset: offset, + }; + + let sink = S3Sink::new(service, request_options, partitioner, batch_settings); + return Ok(VectorSink::from_event_streamsink(sink)); + } + let (framer, serializer) = self.encoding.build(SinkType::MessageBased)?; - let encoder = Encoder::::new(framer, serializer); + let encoder = EncoderKind::Framed(Box::new(Encoder::::new(framer, serializer))); let request_options = S3RequestOptions { bucket: self.bucket.clone(), @@ -289,4 +363,260 @@ mod tests { fn generate_config() { crate::test_util::test_generate_config::(); } + + /// Correct TOML shape: `batch_encoding.codec = "parquet"` with `schema_mode = "auto_infer"`. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_batch_encoding_correct_toml_shape() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + schema_mode = "auto_infer" + codec = "parquet" + + [batch_encoding.compression] + algorithm = "snappy" + + "#, + ) + .expect("correct batch_encoding shape should parse"); + + let batch_enc = config + .batch_encoding + .expect("batch_encoding should be Some"); + match batch_enc { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(ref p) => { + use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; + assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); + assert_eq!(p.compression, ParquetCompression::Snappy); + } + #[allow(unreachable_patterns)] + _ => panic!("expected Parquet variant"), + } + } + + /// Content-Type must be auto-detected as `application/vnd.apache.parquet` + /// when `batch_encoding` is set and `content_type` is not explicitly provided. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_content_type_auto_detected() { + use vector_lib::codecs::encoding::format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializerConfig, + }; + + use crate::sinks::s3_common::config::S3Options; + use crate::sinks::util::{BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression}; + use vector_lib::codecs::TextSerializerConfig; + use vector_lib::codecs::encoding::{BatchSerializerConfig, FramingConfig}; + + let parquet_config = ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression: ParquetCompression::Snappy, + ..Default::default() + }; + + let config = S3SinkConfig { + bucket: "test".to_string(), + key_prefix: super::default_key_prefix(), + filename_time_format: super::default_filename_time_format(), + filename_append_uuid: true, + filename_extension: None, + options: S3Options::default(), + region: crate::aws::RegionOrEndpoint::with_both("us-east-1", "http://localhost:4566"), + encoding: (None::, TextSerializerConfig::default()).into(), + batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + compression: Compression::None, + batch: BatchConfig::::default(), + request: Default::default(), + tls: Default::default(), + auth: Default::default(), + acknowledgements: Default::default(), + timezone: Default::default(), + force_path_style: true, + retry_strategy: Default::default(), + }; + + let batch_config = config.batch_encoding.as_ref().unwrap(); + let batch_serializer = batch_config.build_batch_serializer().unwrap(); + let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); + + let mut api_options = config.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = Some(batch_encoder.content_type().to_string()); + } + + assert_eq!( + api_options.content_type.as_deref(), + Some("application/vnd.apache.parquet"), + "Content-Type must be auto-detected for Parquet" + ); + } + + /// When user explicitly sets `content_type`, the auto-detection must not override it. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_content_type_user_override_preserved() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + content_type = "application/octet-stream" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + + [batch_encoding.compression] + algorithm = "gzip" + level = 9 + "#, + ) + .unwrap(); + + let batch_config = config.batch_encoding.as_ref().unwrap(); + let batch_serializer = batch_config.build_batch_serializer().unwrap(); + let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); + + let mut api_options = config.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = Some(batch_encoder.content_type().to_string()); + } + + assert_eq!( + api_options.content_type.as_deref(), + Some("application/octet-stream"), + "User-specified Content-Type must not be overridden" + ); + } + + /// Parquet filename extension defaults to `.parquet` when not explicitly set. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_filename_extension_defaults_to_parquet() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + "#, + ) + .unwrap(); + + assert!( + config.filename_extension.is_none(), + "fixture must not set filename_extension" + ); + + let batch_config = config.batch_encoding.as_ref().unwrap(); + let extension = config + .filename_extension + .clone() + .or_else(|| match batch_config { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(_) => { + Some("parquet".to_string()) + } + #[allow(unreachable_patterns)] + _ => None, + }); + + assert_eq!(extension.as_deref(), Some("parquet")); + } + + /// Explicit filename_extension overrides the `.parquet` default. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_filename_extension_user_override() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + filename_extension = "pq" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + "#, + ) + .unwrap(); + + assert_eq!(config.filename_extension.as_deref(), Some("pq")); + } + + /// `schema_mode` defaults to `relaxed` when not specified. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_schema_mode_defaults_to_relaxed() { + use vector_lib::codecs::encoding::format::ParquetSchemaMode; + + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + "#, + ) + .unwrap(); + + match config.batch_encoding.unwrap() { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { + assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); + } + #[allow(unreachable_patterns)] + _ => panic!("expected Parquet variant"), + } + } + + /// Explicit `schema_mode = "strict"` is correctly parsed. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_schema_mode_strict_parsed() { + use vector_lib::codecs::encoding::format::ParquetSchemaMode; + + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "strict" + schema_file = "tmp/something.schema" + "#, + ) + .unwrap(); + + match config.batch_encoding.unwrap() { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { + assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); + } + #[allow(unreachable_patterns)] + _ => panic!("expected Parquet variant"), + } + } } diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index 31907d70b5c0f..ff2e0e5a5146b 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -19,6 +19,8 @@ use flate2::read::MultiGzDecoder; use futures::{Stream, stream}; use similar_asserts::assert_eq; use tokio_stream::StreamExt; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ codecs::{TextSerializerConfig, encoding::FramingConfig}, config::proxy::ProxyConfig, @@ -433,6 +435,8 @@ async fn s3_flush_on_exhaustion() { options: S3Options::default(), region: RegionOrEndpoint::with_both("us-east-1", s3_address()), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::None, batch, request: TowerRequestConfig::default(), @@ -490,6 +494,106 @@ async fn s3_flush_on_exhaustion() { assert_eq!(lines, response_lines); // if all events are received, and lines.len() < batch size, then a flush was performed. } +#[cfg(feature = "codecs-parquet")] +#[tokio::test] +async fn s3_parquet_insert_message() { + use vector_lib::codecs::encoding::format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializerConfig, + }; + + let cx = SinkContext::default(); + let bucket = uuid::Uuid::new_v4().to_string(); + create_bucket(&bucket, false).await; + + let parquet_config = ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression: ParquetCompression::Snappy, + ..Default::default() + }; + + let mut batch = BatchConfig::default(); + batch.max_events = Some(100); + batch.timeout_secs = Some(5.0); + + let config = S3SinkConfig { + bucket: bucket.to_string(), + key_prefix: random_string(10) + "/date=%F", + filename_time_format: default_filename_time_format(), + filename_append_uuid: true, + filename_extension: None, + options: S3Options::default(), + region: RegionOrEndpoint::with_both("us-east-1", s3_address()), + encoding: (None::, TextSerializerConfig::default()).into(), + batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + compression: Compression::None, + batch, + request: TowerRequestConfig::default(), + tls: Default::default(), + auth: Default::default(), + acknowledgements: Default::default(), + timezone: Default::default(), + force_path_style: true, + retry_strategy: Default::default(), + }; + + let prefix = config.key_prefix.clone(); + let service = config.create_service(&cx.globals.proxy).await.unwrap(); + let sink = config.build_processor(service, cx).unwrap(); + + let (batch_notifier, receiver) = BatchNotifier::new_with_receiver(); + let events: Vec = (0..10) + .map(|i| { + let mut log = LogEvent::from(format!("message_{}", i)); + log.insert("host", format!("host_{}", i % 3)); + Event::from(log).with_batch_notifier(&batch_notifier) + }) + .collect(); + + drop(batch_notifier); + run_and_assert_sink_compliance(sink, stream::iter(events), &AWS_SINK_TAGS).await; + assert_eq!(receiver.await, BatchStatus::Delivered); + + let keys = get_keys(&bucket, prefix).await; + assert_eq!(keys.len(), 1); + + let key = keys[0].clone(); + assert!( + key.ends_with(".parquet"), + "Expected .parquet extension, got: {}", + key + ); + + // Download and validate Parquet file + let obj = get_object(&bucket, key).await; + assert_eq!(obj.content_encoding, None); + + let body = obj.body.collect().await.unwrap().into_bytes(); + assert!(body.len() >= 4, "Output too short to be valid Parquet"); + assert_eq!(&body[..4], b"PAR1", "Missing Parquet magic bytes"); + + // Verify we can read rows from the Parquet file + use bytes::Bytes; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::reader::RowIter; + + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(&body)).expect("Invalid Parquet file"); + let row_count = RowIter::from_file_into(Box::new(reader)).count(); + assert_eq!(row_count, 10, "Expected 10 rows in Parquet file"); + + // Verify schema has our columns + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(&body)).expect("Invalid Parquet file"); + let schema = reader.metadata().file_metadata().schema_descr(); + let columns: Vec = schema + .columns() + .iter() + .map(|c| c.name().to_string()) + .collect(); + assert!(columns.contains(&"message".to_string())); + assert!(columns.contains(&"host".to_string())); +} + async fn client() -> S3Client { let auth = AwsAuthentication::test_auth(); let region = RegionOrEndpoint::with_both("us-east-1", s3_address()); @@ -526,6 +630,8 @@ fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { options: S3Options::default(), region: RegionOrEndpoint::with_both("us-east-1", s3_address()), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::None, batch, request: TowerRequestConfig::default(), diff --git a/src/sinks/aws_s3/sink.rs b/src/sinks/aws_s3/sink.rs index 26d47cdb7039c..442e0beb544b6 100644 --- a/src/sinks/aws_s3/sink.rs +++ b/src/sinks/aws_s3/sink.rs @@ -3,10 +3,10 @@ use std::io; use bytes::Bytes; use chrono::{FixedOffset, Utc}; use uuid::Uuid; -use vector_lib::{codecs::encoding::Framer, event::Finalizable, request_metadata::RequestMetadata}; +use vector_lib::{codecs::EncoderKind, event::Finalizable, request_metadata::RequestMetadata}; use crate::{ - codecs::{Encoder, Transformer}, + codecs::Transformer, event::Event, sinks::{ s3_common::{ @@ -28,7 +28,7 @@ pub struct S3RequestOptions { pub filename_append_uuid: bool, pub filename_extension: Option, pub api_options: S3Options, - pub encoder: (Transformer, Encoder), + pub encoder: (Transformer, EncoderKind), pub compression: Compression, pub filename_tz_offset: Option, } @@ -36,7 +36,7 @@ pub struct S3RequestOptions { impl RequestBuilder<(S3PartitionKey, Vec)> for S3RequestOptions { type Metadata = S3Metadata; type Events = Vec; - type Encoder = (Transformer, Encoder); + type Encoder = (Transformer, EncoderKind); type Payload = Bytes; type Request = S3Request; type Error = io::Error; // TODO: this is ugly. diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 58ed0a0ac3445..d7af3413c6b97 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -279,7 +279,7 @@ impl ClickhouseConfig { }; if let Some(batch_encoding) = &self.batch_encoding { - use vector_lib::codecs::{BatchEncoder, BatchSerializer}; + use vector_lib::codecs::BatchEncoder; // Validate that batch_encoding is only compatible with ArrowStream format if self.format != Format::ArrowStream { @@ -292,6 +292,13 @@ impl ClickhouseConfig { let mut arrow_config = match batch_encoding { BatchSerializerConfig::ArrowStream(config) => config.clone(), + #[cfg(feature = "codecs-parquet")] + BatchSerializerConfig::Parquet(_) => { + return Err( + "ClickHouse sink does not support Parquet batch encoding. Use 'arrow_stream' instead." + .into(), + ); + } }; self.resolve_arrow_schema( @@ -304,8 +311,7 @@ impl ClickhouseConfig { .await?; let resolved_batch_config = BatchSerializerConfig::ArrowStream(arrow_config); - let arrow_serializer = resolved_batch_config.build()?; - let batch_serializer = BatchSerializer::Arrow(arrow_serializer); + let batch_serializer = resolved_batch_config.build_batch_serializer()?; let encoder = EncoderKind::Batch(BatchEncoder::new(batch_serializer)); return Ok((Format::ArrowStream, encoder)); diff --git a/website/cue/reference/components/sinks/aws_s3.cue b/website/cue/reference/components/sinks/aws_s3.cue index cec5a50e47f82..6f42d082959ba 100644 --- a/website/cue/reference/components/sinks/aws_s3.cue +++ b/website/cue/reference/components/sinks/aws_s3.cue @@ -214,6 +214,135 @@ components: sinks: aws_s3: components._aws & { `storage_class` option. """ } + + parquet_encoding: { + title: "Parquet Batch Encoding" + body: """ + The S3 sink supports Apache Parquet batch encoding with the `batch_encoding` + option. When configured, events are encoded together as Parquet columnar files + instead of the default per-event JSON or text encoding. Parquet files are + optimized for analytical queries using Athena, Trino, Spark, and other columnar + query engines. + + Parquet handles compression internally at the column page level, so the + top-level `compression` setting must be set to `"none"`. + + Output files automatically use the `.parquet` extension. + + This feature requires the `codecs-parquet` feature flag at compile time. + + There are two ways to provide a schema: supply a `schema_file`, or set + `schema_mode` to `auto_infer` and let Vector derive the schema from each + incoming batch. + + #### Option 1: Schema file + + Load the schema from a native Parquet `.schema` file. The file must + contain a valid Parquet message type definition. + + ```toml + [sinks.s3_parquet] + type = "aws_s3" + inputs = ["my-source"] + bucket = "my-analytics-bucket" + key_prefix = "logs/date=%F" + compression = "none" + + [sinks.s3_parquet.encoding] + codec = "text" + + [sinks.s3_parquet.batch_encoding] + codec = "parquet" + schema_file = "/etc/vector/schemas/logs.schema" + schema_mode = "relaxed" + + [sinks.s3_parquet.batch_encoding.compression] + algorithm = "snappy" + ``` + + #### Option 2: Auto-infer schema + + Vector infers the Arrow schema from the fields present in each batch. + `Value::Timestamp` fields are automatically promoted to + `Timestamp(Microsecond, UTC)`. No schema file is required. + + ```toml + [sinks.s3_parquet] + type = "aws_s3" + inputs = ["my-source"] + bucket = "my-analytics-bucket" + key_prefix = "logs/date=%F" + compression = "none" + + [sinks.s3_parquet.encoding] + codec = "text" + + [sinks.s3_parquet.batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + + [sinks.s3_parquet.batch_encoding.compression] + algorithm = "snappy" + ``` + + #### YAML example + + ```yaml + sinks: + s3_parquet: + type: aws_s3 + inputs: + - my-source + bucket: my-analytics-bucket + key_prefix: "logs/date=%F" + compression: none + encoding: + codec: text + batch_encoding: + codec: parquet + schema_mode: auto_infer + compression: + algorithm: gzip + level: 9 + ``` + + #### Configuration reference + + | Field | Type | Required | Description | + |---|---|---|---| + | `codec` | string | yes | Must be `"parquet"` | + | `schema_file` | path | no | Path to a native Parquet `.schema` file. Required when `schema_mode` is `relaxed` or `strict`. | + | `schema_mode` | string | no | `relaxed` (default), `strict`, or `auto_infer`. See the section on schema_mode values. | + | `compression` | object | no | Column-level compression. See the section on compression options. | + + #### `schema_mode` values + + | Value | Description | + |---|---| + | `relaxed` (default) | Missing schema fields become null. Extra event fields are silently dropped. | + | `strict` | Missing schema fields become null. Extra event fields cause an encoding error. | + | `auto_infer` | Schema is inferred from each batch. No `schema_file` needed. `Value::Timestamp` fields are promoted to `Timestamp(Microsecond, UTC)`. | + + #### Compression options + + Compression is configured as a nested object with an `algorithm` key. + Algorithms that support levels accept an additional `level` key. + + | Algorithm | Level range | Default | + |---|---|---| + | `snappy` | — | yes | + | `zstd` | 1–21 | — | + | `gzip` | 1–9 | — | + | `lz4` | — | — | + | `none` | — | — | + + #### Unsupported types + + Binary fields are rejected at config time because the internal Arrow JSON + encoder cannot materialize them. Use `utf8` with base64 or hex encoding + for binary data instead. + """ + } } permissions: iam: [ From 51121047d3201394c2958e4a66696f3aef2d3f11 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 13:25:02 -0400 Subject: [PATCH 079/364] chore(deps): bump rsa from 0.9.3 to 0.9.10 (#25198) * chore(deps): bump rsa from 0.9.3 to 0.9.10 * Update licenses --- Cargo.lock | 17 +++++++++-------- LICENSE-3rdparty.csv | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae4c753529775..d751a3e296d54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3590,15 +3590,16 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ "curve25519-dalek", "ed25519", "serde", "sha2", "signature", + "subtle", "zeroize", ] @@ -9323,9 +9324,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.3" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ef35bf3e7fe15a53c4ab08a998e42271eab13eb0db224126bc7bc4c4bad96d" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", "digest", @@ -10159,9 +10160,9 @@ dependencies = [ [[package]] name = "signature" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe458c98333f9c8152221191a77e2a44e8325d0193484af2e9421a53019e57d" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", "rand_core 0.6.4", @@ -10398,9 +10399,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", "der", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ac6f8bc2acb4a..ed18e3c3c172b 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -242,7 +242,7 @@ dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 OR MIT,RustCrypto Developers -ed25519-dalek,https://github.com/dalek-cryptography/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " +ed25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " educe,https://github.com/magiclen/educe,MIT,Magic Len either,https://github.com/bluss/either,MIT OR Apache-2.0,bluss elliptic-curve,https://github.com/RustCrypto/traits/tree/master/elliptic-curve,Apache-2.0 OR MIT,RustCrypto Developers From dac7c31501defdf451f9422b6a569358fb5ec19e Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 13:53:42 -0400 Subject: [PATCH 080/364] chore(deps): bump rustls-webpki 0.103.10 to 0.103.12 (RUSTSEC-2026-0099) (#25200) * chore(deps): bump rustls-webpki 0.103.10 to 0.103.12 (RUSTSEC-2026-0099) * Add RUSTSEC-2026-0099 to deny.toml * Add RUSTSEC-2026-0098 to deny.toml --- Cargo.lock | 6 +++--- deny.toml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d751a3e296d54..dc20d39632c7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9492,7 +9492,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.10", + "rustls-webpki 0.103.12", "subtle", "zeroize", ] @@ -9574,9 +9574,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "ring", "rustls-pki-types", diff --git a/deny.toml b/deny.toml index c19b0a23f5ee6..ffc2f8b68f240 100644 --- a/deny.toml +++ b/deny.toml @@ -44,5 +44,7 @@ ignore = [ { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0098", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, ] From 6818b9c348cd4c3edd39f9f303f66362676e47a3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 15 Apr 2026 14:22:40 -0400 Subject: [PATCH 081/364] feat(ci): add work-in-progress label workflow for docs PRs (#24950) * feat(ci): add workflow to manage work-in-progress label on PRs Automatically adds a "work in progress" label when a PR is opened or reopened, and removes it when a member or owner approves the PR. Co-Authored-By: Claude Sonnet 4.6 * feat(ci): replace work-in-progress with under_review label for docs PRs Target only PRs that touch docs-owned paths (matching CODEOWNERS) so the docs team isn't pinged until a Vector team member approves. The label is auto-added on open/reopen and auto-removed on first member approval. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(ci): fix prettier formatting Co-Authored-By: Claude Opus 4.6 (1M context) * feat(ci): use work-in-progress label instead of under_review Co-Authored-By: Claude Opus 4.6 (1M context) * feat(ci): split wip label into add/remove workflows with tested approach - Split single workflow into add_wip_label.yml and remove_wip_label.yml to avoid paths filter affecting pull_request_review events - Move association and label checks into github-script step instead of job if condition (GitHub Actions silently fails author_association checks in if expressions) - Tested end-to-end in vectordotdev/ci-sandbox Co-Authored-By: Claude Opus 4.6 (1M context) * feat(ci): trigger WIP label on synchronize to catch docs paths added later Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 --- .github/CODEOWNERS | 1 + .github/workflows/add_wip_label.yml | 22 +++++++++++++ .github/workflows/remove_wip_label.yml | 45 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 .github/workflows/add_wip_label.yml create mode 100644 .github/workflows/remove_wip_label.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ced3f0fb7bc07..ecd6fb374af9c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,7 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance +# Keep documentation team paths in sync with .github/workflows/add_wip_label.yml docs/ @vectordotdev/vector @vectordotdev/documentation website/ @vectordotdev/vector website/content @vectordotdev/vector @vectordotdev/documentation diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_wip_label.yml new file mode 100644 index 0000000000000..a07f9ca0921fa --- /dev/null +++ b/.github/workflows/add_wip_label.yml @@ -0,0 +1,22 @@ +name: Add Work In Progress Label + +permissions: + pull-requests: write + +on: + pull_request_target: + types: [opened, reopened, synchronize] + # Keep these paths in sync with the documentation team entries in .github/CODEOWNERS + paths: + - "docs/**" + - "website/content/**" + - "website/cue/reference/**" + +jobs: + add_wip_label: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 + with: + labels: "work in progress" diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_wip_label.yml new file mode 100644 index 0000000000000..46d6e79cf1ad8 --- /dev/null +++ b/.github/workflows/remove_wip_label.yml @@ -0,0 +1,45 @@ +name: Remove Work In Progress Label + +permissions: + pull-requests: write + +on: + pull_request_review: + types: [submitted] + +jobs: + remove_wip_label: + if: github.event.review.state == 'approved' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check association, label, and remove + uses: actions/github-script@v7 + with: + script: | + const association = context.payload.review.author_association; + core.info(`Author association: ${association}`); + + const allowed = ['MEMBER', 'OWNER']; + if (!allowed.includes(association)) { + core.info(`Association "${association}" not in allowed list, skipping`); + return; + } + + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + }); + const hasLabel = labels.some(l => l.name === 'work in progress'); + if (!hasLabel) { + core.info('Label "work in progress" not found, skipping'); + return; + } + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + name: 'work in progress', + }); + core.info('Removed "work in progress" label'); From 2d14034bf8115764c52636a59ec298d7444a4d28 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 14:52:21 -0400 Subject: [PATCH 082/364] fix(deps): override smol-toml to 1.6.1 for markdownlint-cli2 (#25202) fix(deps): override smol-toml to 1.6.1 for markdownlint-cli2 (GHSA-v3rj-xjv7-4jmq) --- scripts/environment/npm-tools/package-lock.json | 6 +++--- scripts/environment/npm-tools/package.json | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index d9e098652b6ad..ac93d1c99990a 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -1232,9 +1232,9 @@ } }, "node_modules/smol-toml": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", - "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", + "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", "license": "BSD-3-Clause", "engines": { "node": ">= 18" diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json index a40c7f7c3e2ae..fdffa6f8a2ac0 100644 --- a/scripts/environment/npm-tools/package.json +++ b/scripts/environment/npm-tools/package.json @@ -1,6 +1,11 @@ { "private": true, "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", + "overrides": { + "markdownlint-cli2": { + "smol-toml": "1.6.1" + } + }, "dependencies": { "@datadog/datadog-ci": "5.13.0", "markdownlint-cli2": "0.22.0", From 08c9f1e6b54cf1b30e88d536f3545e74ff193c0f Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 15:48:46 -0400 Subject: [PATCH 083/364] chore(deps): bump VRL and add feature for enable_crypto_functions (#25205) --- Cargo.toml | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6b233580143c8..84ded822b5790 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -533,6 +533,7 @@ tokio-console = ["dep:console-subscriber", "tokio/tracing"] vrl-functions-env = ["vrl/enable_env_functions"] vrl-functions-system = ["vrl/enable_system_functions"] vrl-functions-network = ["vrl/enable_network_functions"] +vrl-functions-crypto = ["vrl/enable_crypto_functions"] # Enables the binary secret-backend-example secret-backend-example = ["transforms"] diff --git a/Makefile b/Makefile index 34da1302487d9..821ff961de4ac 100644 --- a/Makefile +++ b/Makefile @@ -373,7 +373,7 @@ test-behavior-config: ## Runs configuration related behavioral tests .PHONY: test-behavior-% test-behavior-%: ## Runs behavioral test for a given category - ${MAYBE_ENVIRONMENT_EXEC} cargo run --no-default-features --features transforms,vrl-functions-env,vrl-functions-system,vrl-functions-network -- test tests/behavior/$*/* + ${MAYBE_ENVIRONMENT_EXEC} cargo run --no-default-features --features transforms,vrl-functions-env,vrl-functions-system,vrl-functions-network,vrl-functions-crypto -- test tests/behavior/$*/* .PHONY: test-behavior test-behavior: ## Runs all behavioral tests From a1469590bed6996b1eb7001248232b0aa48cd777 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 16:06:08 -0400 Subject: [PATCH 084/364] chore(deps): bump rand 0.10.0 to 0.10.1 and 0.9.2 to 0.9.4 (#25204) * chore(deps): bump rand 0.10.0 to 0.10.1 (RUSTSEC-2026-0097) * chore(deps): bump rand 0.9.2 to 0.9.4 (RUSTSEC-2026-0097) --- Cargo.lock | 64 +++++++++++++++++++++++++++--------------------------- deny.toml | 1 + 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc20d39632c7c..aadbb2dd580a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.9.2", + "rand 0.9.4", "regex-lite", "serde", "serde_bytes", @@ -2022,7 +2022,7 @@ dependencies = [ "indexmap 2.12.0", "js-sys", "once_cell", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_bytes", "serde_json", @@ -2446,7 +2446,7 @@ dependencies = [ "pin-project", "prost 0.12.6", "prost-reflect", - "rand 0.9.2", + "rand 0.9.4", "regex", "rstest", "rust_decimal", @@ -3910,7 +3910,7 @@ version = "0.1.0" dependencies = [ "chrono", "fakedata_generator", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -3920,7 +3920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42ec18312c068f0d6008ad66ad2fa289ab40a16d07a06406c200baa09eb1e7c" dependencies = [ "passt", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", ] @@ -4412,7 +4412,7 @@ dependencies = [ "nonzero_ext", "parking_lot", "portable-atomic", - "rand 0.9.2", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -4445,7 +4445,7 @@ dependencies = [ "greptime-proto", "parking_lot", "prost 0.12.6", - "rand 0.9.2", + "rand 0.9.4", "snafu 0.8.9", "tokio", "tokio-stream", @@ -4819,7 +4819,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustls-pki-types", "thiserror 2.0.17", @@ -4864,7 +4864,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "resolv-conf", "smallvec", "thiserror 2.0.17", @@ -5794,7 +5794,7 @@ dependencies = [ "indoc", "k8s-openapi", "k8s-test-framework", - "rand 0.9.2", + "rand 0.9.4", "regex", "reqwest 0.11.26", "serde_json", @@ -7942,7 +7942,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.9.2", + "rand 0.9.4", "sha2", "stringprep", ] @@ -8172,7 +8172,7 @@ dependencies = [ "bit-vec", "bitflags 2.10.0", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -8251,7 +8251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.5.0", + "heck 0.4.1", "itertools 0.12.1", "log", "multimap", @@ -8271,7 +8271,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "itertools 0.14.0", "log", "multimap", @@ -8528,7 +8528,7 @@ checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" dependencies = [ "env_logger", "log", - "rand 0.10.0", + "rand 0.10.1", ] [[package]] @@ -8569,7 +8569,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls 0.23.37", @@ -8660,9 +8660,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.0", @@ -8670,9 +8670,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", "getrandom 0.4.2", @@ -8731,7 +8731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -8900,7 +8900,7 @@ dependencies = [ "num-bigint", "percent-encoding", "pin-project-lite", - "rand 0.9.2", + "rand 0.9.4", "ryu", "socket2 0.6.0", "tokio", @@ -9211,7 +9211,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46a4bd6027df676bcb752d3724db0ea3c0c5fc1dd0376fec51ac7dcaf9cc69be" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -10323,7 +10323,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -10335,7 +10335,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -11262,7 +11262,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.2", + "rand 0.9.4", "socket2 0.6.0", "tokio", "tokio-util", @@ -11987,7 +11987,7 @@ dependencies = [ "futures", "getrandom 0.3.4", "pin-project", - "rand 0.9.2", + "rand 0.9.4", "reqwest 0.12.28", "serde", "serde_json", @@ -12231,7 +12231,7 @@ checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ "getrandom 0.4.2", "js-sys", - "rand 0.10.0", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] @@ -12428,7 +12428,7 @@ dependencies = [ "quick-junit", "quick-xml 0.31.0", "quickcheck", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "rdkafka", "redis", @@ -12549,7 +12549,7 @@ dependencies = [ "pastey", "proptest", "quickcheck", - "rand 0.9.2", + "rand 0.9.4", "rkyv", "serde", "serde_yaml", @@ -12698,7 +12698,7 @@ dependencies = [ "quanta", "quickcheck", "quickcheck_macros", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "regex", "ryu", @@ -12775,7 +12775,7 @@ dependencies = [ "futures-util", "pin-project", "proptest", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "tokio", "tokio-util", diff --git a/deny.toml b/deny.toml index ffc2f8b68f240..ac8e588fa1cbe 100644 --- a/deny.toml +++ b/deny.toml @@ -47,4 +47,5 @@ ignore = [ { id = "RUSTSEC-2026-0098", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, + { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 unsound with custom logger - transitive dependency, upstream crates have not updated to rand 0.9+" }, ] From 5ba8405bf48cd64509bf02388c7901451bbe67f4 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 16:20:05 -0400 Subject: [PATCH 085/364] chore(vdev): bump version to 0.3.1 (#25206) --- Cargo.lock | 2 +- vdev/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aadbb2dd580a9..2d641cc2a2934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12260,7 +12260,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "cfg-if", diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 7de3fc06d5a59..2a0f6169a160e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.0" +version = "0.3.1" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0" From ce651754d4e2eb134ee9ab2fe85a8a3a5ed923a0 Mon Sep 17 00:00:00 2001 From: Yoenn Burban <62966490+gwenaskell@users.noreply.github.com> Date: Thu, 16 Apr 2026 09:16:53 +0200 Subject: [PATCH 086/364] feat(sources): add source latency metric and fix source lag time on large batches (#24987) * add send latency distribution to source sender * add changelog * use outputMetrics in the builder * fix clippy * apply suggestion * only record the await point duration * make sure metrics are updated if an error is returned * add documentation for the added metrics * fix source_lag_time_seconds measurement for large batches --- ...fix_source_lag_time_chunked_batches.fix.md | 3 + ...urce_sender_latency_metrics.enhancement.md | 5 ++ lib/vector-core/src/source_sender/builder.rs | 20 +++-- lib/vector-core/src/source_sender/mod.rs | 4 +- lib/vector-core/src/source_sender/output.rs | 87 +++++++++++++++---- lib/vector-core/src/source_sender/sender.rs | 19 +++- .../components/sources/internal_metrics.cue | 12 +++ 7 files changed, 122 insertions(+), 28 deletions(-) create mode 100644 changelog.d/fix_source_lag_time_chunked_batches.fix.md create mode 100644 changelog.d/source_sender_latency_metrics.enhancement.md diff --git a/changelog.d/fix_source_lag_time_chunked_batches.fix.md b/changelog.d/fix_source_lag_time_chunked_batches.fix.md new file mode 100644 index 0000000000000..6303ba10edd33 --- /dev/null +++ b/changelog.d/fix_source_lag_time_chunked_batches.fix.md @@ -0,0 +1,3 @@ +Fixed an incorrect source_lag_time_seconds measurement in sources that use `send_batch` with large event batches. When a batch was split into multiple chunks, the reference timestamp used to compute lag time was re-captured on each chunk send, causing the lag time for later chunks to be overstated by the amount of time spent waiting for the channel to accept earlier chunks. The reference timestamp is now captured once before iteration and shared across all chunks. + +authors: gwenaskell diff --git a/changelog.d/source_sender_latency_metrics.enhancement.md b/changelog.d/source_sender_latency_metrics.enhancement.md new file mode 100644 index 0000000000000..6876641a37764 --- /dev/null +++ b/changelog.d/source_sender_latency_metrics.enhancement.md @@ -0,0 +1,5 @@ +Sources now record the distribution metrics `source_send_latency_seconds` (measuring the time spent +blocking on a single events chunk send operation on the output) and `source_send_batch_latency_seconds` +(encompassing all chunks within a received events batch). + +authors: gwenaskell diff --git a/lib/vector-core/src/source_sender/builder.rs b/lib/vector-core/src/source_sender/builder.rs index ec16d54d482a9..f3051726a7f56 100644 --- a/lib/vector-core/src/source_sender/builder.rs +++ b/lib/vector-core/src/source_sender/builder.rs @@ -1,17 +1,20 @@ use std::{collections::HashMap, time::Duration}; -use metrics::{Histogram, histogram}; +use metrics::histogram; use vector_buffers::topology::channel::LimitedReceiver; use vector_common::internal_event::DEFAULT_OUTPUT; -use super::{CHUNK_SIZE, LAG_TIME_NAME, Output, SourceSender, SourceSenderItem}; +use super::{ + CHUNK_SIZE, LAG_TIME_NAME, Output, OutputMetrics, SEND_BATCH_LATENCY_NAME, SEND_LATENCY_NAME, + SourceSender, SourceSenderItem, +}; use crate::config::{ComponentKey, OutputId, SourceOutput}; pub struct Builder { buf_size: usize, default_output: Option, named_outputs: HashMap, - lag_time: Option, + output_metrics: OutputMetrics, timeout: Option, ewma_half_life_seconds: Option, } @@ -22,7 +25,11 @@ impl Default for Builder { buf_size: CHUNK_SIZE, default_output: None, named_outputs: Default::default(), - lag_time: Some(histogram!(LAG_TIME_NAME)), + output_metrics: OutputMetrics::new( + Some(histogram!(LAG_TIME_NAME)), + Some(histogram!(SEND_LATENCY_NAME)), + Some(histogram!(SEND_BATCH_LATENCY_NAME)), + ), timeout: None, ewma_half_life_seconds: None, } @@ -53,7 +60,6 @@ impl Builder { output: SourceOutput, component_key: ComponentKey, ) -> LimitedReceiver { - let lag_time = self.lag_time.clone(); let log_definition = output.schema_definition.clone(); let output_id = OutputId { component: component_key, @@ -64,7 +70,7 @@ impl Builder { let (output, rx) = Output::new_with_buffer( self.buf_size, DEFAULT_OUTPUT.to_owned(), - lag_time, + self.output_metrics.clone(), log_definition, output_id, self.timeout, @@ -77,7 +83,7 @@ impl Builder { let (output, rx) = Output::new_with_buffer( self.buf_size, name.clone(), - lag_time, + self.output_metrics.clone(), log_definition, output_id, self.timeout, diff --git a/lib/vector-core/src/source_sender/mod.rs b/lib/vector-core/src/source_sender/mod.rs index c8af2db8bbf87..fc0cb417890a0 100644 --- a/lib/vector-core/src/source_sender/mod.rs +++ b/lib/vector-core/src/source_sender/mod.rs @@ -14,7 +14,7 @@ mod tests; pub use builder::Builder; pub use errors::SendError; -use output::Output; +use output::{Output, OutputMetrics}; pub use sender::{SourceSender, SourceSenderItem}; pub const CHUNK_SIZE: usize = 1000; @@ -23,3 +23,5 @@ pub const CHUNK_SIZE: usize = 1000; const TEST_BUFFER_SIZE: usize = 100; const LAG_TIME_NAME: &str = "source_lag_time_seconds"; +const SEND_LATENCY_NAME: &str = "source_send_latency_seconds"; +const SEND_BATCH_LATENCY_NAME: &str = "source_send_batch_latency_seconds"; diff --git a/lib/vector-core/src/source_sender/output.rs b/lib/vector-core/src/source_sender/output.rs index df0bee2a43da0..4a194cc3e1c1f 100644 --- a/lib/vector-core/src/source_sender/output.rs +++ b/lib/vector-core/src/source_sender/output.rs @@ -85,7 +85,7 @@ impl Drop for UnsentEventCount { #[derive(Clone)] pub(super) struct Output { sender: LimitedSender, - lag_time: Option, + metrics: OutputMetrics, events_sent: Registered, /// The schema definition that will be attached to Log events sent through here log_definition: Option>, @@ -95,6 +95,27 @@ pub(super) struct Output { timeout: Option, } +#[derive(Clone, Default)] +pub(super) struct OutputMetrics { + lag_time: Option, + send_latency: Option, + send_batch_latency: Option, +} + +impl OutputMetrics { + pub(super) fn new( + lag_time: Option, + send_latency: Option, + send_batch_latency: Option, + ) -> Self { + Self { + lag_time, + send_latency, + send_batch_latency, + } + } +} + #[expect(clippy::missing_fields_in_debug)] impl fmt::Debug for Output { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -111,19 +132,20 @@ impl Output { pub(super) fn new_with_buffer( n: usize, output: String, - lag_time: Option, + metrics: OutputMetrics, log_definition: Option>, output_id: OutputId, timeout: Option, ewma_half_life_seconds: Option, ) -> (Self, LimitedReceiver) { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(n).unwrap()); - let metrics = ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); - let (tx, rx) = channel::limited(limit, Some(metrics), ewma_half_life_seconds); + let channel_metrics = + ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); + let (tx, rx) = channel::limited(limit, Some(channel_metrics), ewma_half_life_seconds); ( Self { sender: tx, - lag_time, + metrics, events_sent: internal_event::register(EventsSent::from(internal_event::Output( Some(output.into()), ))), @@ -136,12 +158,21 @@ impl Output { } pub(super) async fn send( + &mut self, + events: EventArray, + unsent_event_count: &mut UnsentEventCount, + ) -> Result<(), SendError> { + let reference = Utc::now().timestamp_millis(); + self.send_inner(events, unsent_event_count, reference).await + } + + async fn send_inner( &mut self, mut events: EventArray, unsent_event_count: &mut UnsentEventCount, + reference: i64, ) -> Result<(), SendError> { let send_reference = Instant::now(); - let reference = Utc::now().timestamp_millis(); events .iter_events() .for_each(|event| self.emit_lag_time(event, reference)); @@ -156,7 +187,17 @@ impl Output { let byte_size = events.estimated_json_encoded_size_of(); let count = events.len(); - self.send_with_timeout(events, send_reference).await?; + + let send_start = Instant::now(); + + let send_result = self.send_with_timeout(events, send_reference).await; + + if let Some(send_latency) = &self.metrics.send_latency { + send_latency.record(send_start.elapsed().as_secs_f64()); + } + + send_result?; + self.events_sent.emit(CountByteSize(count, byte_size)); unsent_event_count.decr(count); Ok(()) @@ -218,25 +259,39 @@ impl Output { I: IntoIterator, ::IntoIter: ExactSizeIterator, { + // Capture a single reference timestamp for the entire batch so that lag time + // measurements are not inflated by channel-send latency for later chunks. + let reference = Utc::now().timestamp_millis(); + // It's possible that the caller stops polling this future while it is blocked waiting // on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit // `ComponentEventsDropped` events. let events = events.into_iter().map(Into::into); let mut unsent_event_count = UnsentEventCount::new(events.len()); + let send_batch_start = Instant::now(); + for events in array::events_into_arrays(events, Some(CHUNK_SIZE)) { - self.send(events, &mut unsent_event_count) + self.send_inner(events, &mut unsent_event_count, reference) .await - .inspect_err(|error| match error { - SendError::Timeout => { - unsent_event_count.timed_out(); + .inspect_err(|error| { + match error { + SendError::Timeout => { + unsent_event_count.timed_out(); + } + SendError::Closed => { + // The unsent event count is discarded here because the callee emits the + // `StreamClosedError`. + unsent_event_count.discard(); + } } - SendError::Closed => { - // The unsent event count is discarded here because the callee emits the - // `StreamClosedError`. - unsent_event_count.discard(); + if let Some(send_batch_latency) = &self.metrics.send_batch_latency { + send_batch_latency.record(send_batch_start.elapsed().as_secs_f64()); } })?; } + if let Some(send_batch_latency) = &self.metrics.send_batch_latency { + send_batch_latency.record(send_batch_start.elapsed().as_secs_f64()); + } Ok(()) } @@ -244,7 +299,7 @@ impl Output { /// timestamp stored in the given event reference, and emit the /// different, as expressed in milliseconds, as a histogram. pub(super) fn emit_lag_time(&self, event: EventRef<'_>, reference: i64) { - if let Some(lag_time_metric) = &self.lag_time { + if let Some(lag_time_metric) = &self.metrics.lag_time { let timestamp = match event { EventRef::Log(log) => { log_schema() diff --git a/lib/vector-core/src/source_sender/sender.rs b/lib/vector-core/src/source_sender/sender.rs index 88cb50172c70a..6041dad3dad9f 100644 --- a/lib/vector-core/src/source_sender/sender.rs +++ b/lib/vector-core/src/source_sender/sender.rs @@ -22,7 +22,9 @@ use vector_common::{ use super::{Builder, Output, SendError}; #[cfg(any(test, feature = "test"))] -use super::{LAG_TIME_NAME, TEST_BUFFER_SIZE}; +use super::{ + LAG_TIME_NAME, OutputMetrics, SEND_BATCH_LATENCY_NAME, SEND_LATENCY_NAME, TEST_BUFFER_SIZE, +}; use crate::{ EstimatedJsonEncodedSizeOf, event::{Event, EventArray, EventContainer, array::EventArrayIntoIter}, @@ -108,6 +110,8 @@ impl SourceSender { timeout: Option, ) -> (Self, LimitedReceiver) { let lag_time = Some(histogram!(LAG_TIME_NAME)); + let send_latency = Some(histogram!(SEND_LATENCY_NAME)); + let send_batch_latency = Some(histogram!(SEND_BATCH_LATENCY_NAME)); let output_id = OutputId { component: "test".to_string().into(), port: None, @@ -115,7 +119,7 @@ impl SourceSender { let (default_output, rx) = Output::new_with_buffer( n, DEFAULT_OUTPUT.to_owned(), - lag_time, + OutputMetrics::new(lag_time, send_latency, send_batch_latency), None, output_id, timeout, @@ -192,8 +196,15 @@ impl SourceSender { component: "test".to_string().into(), port: Some(name.clone()), }; - let (output, recv) = - Output::new_with_buffer(100, name.clone(), None, None, output_id, None, None); + let (output, recv) = Output::new_with_buffer( + 100, + name.clone(), + OutputMetrics::default(), + None, + output_id, + None, + None, + ); let recv = recv.into_stream().map(move |mut item| { item.events.iter_events_mut().for_each(|mut event| { let metadata = event.metadata_mut(); diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 98ed1ffe025d7..1e990965638c6 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -774,6 +774,18 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _component_tags } + source_send_batch_latency_seconds: { + description: "The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source" + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + source_send_latency_seconds: { + description: "The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source" + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } source_buffer_max_byte_size: { description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" From 8138c4593455dd284475ed4ed61fe55ee176bd34 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 16 Apr 2026 11:27:11 -0400 Subject: [PATCH 087/364] chore(website): update DD RUM/Logs config to cose site (#25213) --- website/assets/js/dd-browser-logs-rum.js | 6 ++++++ website/config.toml | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/website/assets/js/dd-browser-logs-rum.js b/website/assets/js/dd-browser-logs-rum.js index fa9414b4099d2..3ec4e2a2fb085 100644 --- a/website/assets/js/dd-browser-logs-rum.js +++ b/website/assets/js/dd-browser-logs-rum.js @@ -12,10 +12,15 @@ if (datadogRum) { datadogRum.init({ applicationId: '{{ $ddConfig.application_id }}', clientToken: '{{ $ddConfig.client_token }}', + site: '{{ $ddConfig.site }}', env, service: '{{ $ddConfig.service_name }}', version: '{{ $latest }}', + sessionSampleRate: 100, + sessionReplaySampleRate: 20, + trackResources: true, trackInteractions: true, + trackLongTasks: true, internalAnalyticsSubdomain: '8b61d74c' }); } @@ -25,6 +30,7 @@ if (datadogLogs) { if (env === 'preview' || env === 'production') { datadogLogs.init({ clientToken: '{{ $ddConfig.client_token }}', + site: '{{ $ddConfig.site }}', forwardErrorsToLogs: true, env, service: '{{ $ddConfig.service_name }}', diff --git a/website/config.toml b/website/config.toml index f107186e280da..414451d38b57d 100644 --- a/website/config.toml +++ b/website/config.toml @@ -93,8 +93,9 @@ discord = "https://chat.vector.dev" # Datadog configuration [params.datadog_config] -application_id = "0b95923b-b06d-445b-893f-a861e93d6ea3" -client_token = "puba5f23a97d613091ae2ca8c0f4a135af4" +application_id = "894af722-5919-4a62-97fc-33f49bc52803" +client_token = "pubcee1dc56e7f5bd03397cc3624eb7961c" +site = "cose.datadoghq.com" service_name = "vector" ## Menus From 61b4563fc60211feba19dcbf9296872b8640ec94 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 16 Apr 2026 12:19:11 -0400 Subject: [PATCH 088/364] fix(ci): fix secrets in static analysis workflow (#25208) fix(ci): use env context instead of secrets in workflow expression --- .github/workflows/static-analysis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 04ed6b19d9f25..9411f2844e264 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -13,14 +13,17 @@ permissions: jobs: static-analysis: runs-on: ubuntu-latest + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_APP_KEY: ${{ secrets.DD_APP_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Datadog Static Analyzer - if: ${{ secrets.DD_API_KEY != '' }} + if: ${{ env.DD_API_KEY != '' }} uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 with: - dd_api_key: ${{ secrets.DD_API_KEY }} - dd_app_key: ${{ secrets.DD_APP_KEY }} + dd_api_key: ${{ env.DD_API_KEY }} + dd_app_key: ${{ env.DD_APP_KEY }} dd_site: datadoghq.com secrets_enabled: true From 467ef876a286df316ba273e4ad1779f1b08ba392 Mon Sep 17 00:00:00 2001 From: Vladimir Zhuk Date: Thu, 16 Apr 2026 19:18:13 +0200 Subject: [PATCH 089/364] fix(datadog_common sink): preserve site prefix when computing API endpoint (#25211) * fix(common): preserve site prefix when computing API endpoint from intake endpoints The `compute_api_endpoint` regex was stripping site prefixes (e.g. `us3.`, `us5.`, `ap1.`) from Datadog domain names when deriving the API endpoint from intake endpoints. This caused the sink healthcheck to call `https://api.datadoghq.com` instead of the site-specific endpoint (e.g. `https://api.us3.datadoghq.com`), resulting in unintended cross-site egress traffic. The fix moves the optional site prefix inside the regex capture group so it is preserved in the rewritten API URL. Co-Authored-By: Claude Opus 4.6 (1M context) * add changelog fragment and fix formatting Co-Authored-By: Claude Opus 4.6 (1M context) * fix changelog * refactor test --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Pavlos Rontidis --- changelog.d/fix_site_prefix_stripping.fix.md | 3 ++ src/common/datadog.rs | 30 +++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fix_site_prefix_stripping.fix.md diff --git a/changelog.d/fix_site_prefix_stripping.fix.md b/changelog.d/fix_site_prefix_stripping.fix.md new file mode 100644 index 0000000000000..8136664b5b2fd --- /dev/null +++ b/changelog.d/fix_site_prefix_stripping.fix.md @@ -0,0 +1,3 @@ +Fixed the Datadog sink healthcheck endpoint computation to preserve site prefixes (e.g. `us3.`, `us5.`, `ap1.`) when deriving the API URL from intake endpoints. Previously, the healthcheck for site-specific endpoints like `https://http-intake.logs.us3.datadoghq.com` would incorrectly call `https://api.datadoghq.com` instead of `https://api.us3.datadoghq.com`, causing unintended cross-site egress traffic. + +authors: vladimir-dd diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 9b84a2f8351dc..05638b474e940 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -105,7 +105,7 @@ fn compute_api_endpoint(endpoint: &str) -> String { // https://github.com/DataDog/datadog-agent/blob/cdcf0fc809b9ac1cd6e08057b4971c7dbb8dbe30/comp/forwarder/defaultforwarder/forwarder_health.go#L45-L47 // https://github.com/DataDog/datadog-agent/blob/cdcf0fc809b9ac1cd6e08057b4971c7dbb8dbe30/comp/forwarder/defaultforwarder/forwarder_health.go#L188-L190 static DOMAIN_REGEX: LazyLock = LazyLock::new(|| { - Regex::new(r"(?:[a-z]{2}\d\.)?(datadoghq\.[a-z]+|ddog-gov\.com)/*$") + Regex::new(r"((?:[a-z]{2}\d\.)?(?:datadoghq\.[a-z]+|ddog-gov\.com))/*$") .expect("Could not build Datadog domain regex") }); @@ -175,6 +175,27 @@ mod tests { ); } + #[test] + fn preserves_site_prefix_in_api_endpoint() { + for (prefix, tld) in [ + ("us3", "com"), + ("us5", "com"), + ("ap1", "com"), + ("eu1", "eu"), + ] { + assert_eq!( + compute_api_endpoint(&format!( + "https://http-intake.logs.{prefix}.datadoghq.{tld}" + )), + format!("https://api.{prefix}.datadoghq.{tld}") + ); + } + assert_eq!( + compute_api_endpoint("https://1-2-3-observability-pipelines.agent.us3.datadoghq.com"), + "https://api.us3.datadoghq.com" + ); + } + #[test] fn gets_correct_api_base_endpoint() { assert_eq!( @@ -189,5 +210,12 @@ mod tests { get_api_base_endpoint(Some("https://logs.datadoghq.eu"), DD_US_SITE), "https://api.datadoghq.eu" ); + assert_eq!( + get_api_base_endpoint( + Some("https://http-intake.logs.us3.datadoghq.com"), + DD_US_SITE + ), + "https://api.us3.datadoghq.com" + ); } } From 9efe09d254fdad82f5d1239e3eae28348390ddfa Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 16 Apr 2026 20:34:41 -0400 Subject: [PATCH 090/364] fix(ci): modernize cross ubuntu bootstrap script (#25215) * chore(cross): modernize ubuntu bootstrap script for cross-compilation * Pin cross edge images * Install libcurl dev headers for librdkafka build * Configure cmake --- Makefile | 14 +++++++++++ scripts/cross/Dockerfile | 14 +++++++++-- scripts/cross/bootstrap-ubuntu.sh | 39 ++++++++++++++----------------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 821ff961de4ac..847a64069f61a 100644 --- a/Makefile +++ b/Makefile @@ -267,13 +267,27 @@ cross-enable: cargo-install-cross CARGO_HANDLES_FRESHNESS: ${EMPTY} +# Pinned digests for ghcr.io/cross-rs/:edge. +# Source: cross-rs/cross @ f86fd03bb70b4c6802847c18087e21391498b0b4, built 2026-04-10 (Ubuntu 20.04 focal). +# Refresh with: crane digest ghcr.io/cross-rs/:edge +CROSS_DIGEST_x86_64-unknown-linux-gnu := sha256:13f7a68e55cb05a19e840bce65834fc785dc069e0c2218d12b8fdb8f8a1519d5 +CROSS_DIGEST_aarch64-unknown-linux-gnu := sha256:3bf094d22fc4f73c9bdce45ddd7a8bbae349efdbd51b4d4b5ee1bedd8454466b +CROSS_DIGEST_x86_64-unknown-linux-musl := sha256:c59deede3efcd7cb6f6a57641241ba1c63cfe35b7965be09a851242b4209639d +CROSS_DIGEST_aarch64-unknown-linux-musl := sha256:dad492e0f040c6e712d4be9b970c9de5f3b8ef9cde6b9a2b437d56d1dabeb808 +CROSS_DIGEST_armv7-unknown-linux-gnueabihf := sha256:73294ebb06e077e49bbbecfe8f17507e9e0b733a2a1ba23056abcd9c0ba617c9 +CROSS_DIGEST_armv7-unknown-linux-musleabihf := sha256:49bdc9a4cf2f1bcb385389c85be8f43c4399fa6d6fe22883702ef13eb921e443 +CROSS_DIGEST_arm-unknown-linux-gnueabi := sha256:0c70b0e54724bd599dff00a2888f8ea176a5b6c85af47aad9ad25296f63e2967 +CROSS_DIGEST_arm-unknown-linux-musleabi := sha256:0ca8f4afcc29fb5964aa63e482452e8869311a610c5868f22ded400c4e483328 + # GNU Make < 3.82 pattern matching priority depends on the definition order # so cross-image-% must be defined before cross-% .PHONY: cross-image-% cross-image-%: export TRIPLE =$($(strip @):cross-image-%=%) cross-image-%: + $(if $(CROSS_DIGEST_$*),,$(error No CROSS_DIGEST pinned for $*. Add it to the digest table in Makefile.)) $(CONTAINER_TOOL) build \ --build-arg TARGET=${TRIPLE} \ + --build-arg CROSS_DIGEST=$(CROSS_DIGEST_$*) \ --file scripts/cross/Dockerfile \ --tag vector-cross-env:${TRIPLE} \ . diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 80a75caba683c..3abe8cd8b2d11 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -1,13 +1,23 @@ -ARG CROSS_VERSION=0.2.5 ARG TARGET=x86_64-unknown-linux-musl +ARG CROSS_DIGEST -FROM ghcr.io/cross-rs/${TARGET}:${CROSS_VERSION} +FROM ghcr.io/cross-rs/${TARGET}@${CROSS_DIGEST} # Common steps for all targets COPY scripts/cross/bootstrap-ubuntu.sh / COPY scripts/environment/install-protoc.sh / RUN /bootstrap-ubuntu.sh && bash /install-protoc.sh +# Allow cmake to find pre-built dependencies (e.g. curl from curl-sys) that +# land in CMAKE_PREFIX_PATH rather than the cross sysroot. The cross-rs +# toolchain.cmake sets CMAKE_FIND_ROOT_PATH_MODE_{LIBRARY,INCLUDE} to ONLY, +# which makes cmake ignore CMAKE_PREFIX_PATH entries outside the sysroot. +# Switching to BOTH lets FindCURL (and similar modules) locate libraries that +# Cargo crates like curl-sys build from source and expose via CMAKE_PREFIX_PATH. +RUN if [ -f /opt/toolchain.cmake ]; then \ + printf '\n# Allow finding pre-built cargo dependencies via CMAKE_PREFIX_PATH\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)\n' >> /opt/toolchain.cmake; \ + fi + # Relocate libstdc++ for musl targets that need it (TODO: investigate if still required) RUN if [ "$TARGET" = "arm-unknown-linux-musleabi" ]; then \ LIBSTDC=/usr/local/arm-linux-musleabi/lib/libstdc++.a; \ diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 8068d1bcd3eb5..d89ea85f37ca5 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -1,31 +1,26 @@ #!/bin/sh set -o errexit -echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries - -apt-get update -apt-get install -y \ - apt-transport-https \ - gnupg \ - wget - -# we need LLVM >= 3.9 for onig_sys/bindgen - -cat <<-EOF > /etc/apt/sources.list.d/llvm.list -deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main -deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main +export DEBIAN_FRONTEND=noninteractive +export ACCEPT_EULA=Y + +# Configure apt for speed and efficiency +cat > /etc/apt/apt.conf.d/90-vector-optimizations < Date: Fri, 17 Apr 2026 13:32:34 -0400 Subject: [PATCH 091/364] chore(dev): split VERSION-dependent packaging targets into Makefile.packaging (#25101) * chore(build): split VERSION-dependent packaging targets into Makefile.packaging * fix(build): unset TRIPLE before delegating to main Makefile in packaging targets --- Makefile | 113 ++------------------------------------- Makefile.packaging | 130 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 108 deletions(-) create mode 100644 Makefile.packaging diff --git a/Makefile b/Makefile index 847a64069f61a..5421a62eed83e 100644 --- a/Makefile +++ b/Makefile @@ -77,9 +77,6 @@ endif export AWS_ACCESS_KEY_ID ?= "dummy" export AWS_SECRET_ACCESS_KEY ?= "dummy" -# Set version -export VERSION ?= $(shell command -v cargo >/dev/null && $(VDEV) version || echo unknown) - # Set if you are on the CI and actually want the things to happen. (Non-CI users should never set this.) export CI ?= false @@ -571,117 +568,17 @@ build-rustdoc: ## Build Vector's Rustdocs # This command is mostly intended for use by the build process in vectordotdev/vector-rustdoc ${MAYBE_ENVIRONMENT_EXEC} cargo doc --no-deps --workspace -##@ Packaging +##@ Packaging (forwarded to Makefile.packaging) -# archives -target/artifacts/vector-${VERSION}-%.tar.gz: export TRIPLE :=$(@:target/artifacts/vector-${VERSION}-%.tar.gz=%) -target/artifacts/vector-${VERSION}-%.tar.gz: override PROFILE =release -target/artifacts/vector-${VERSION}-%.tar.gz: target/%/release/vector.tar.gz - @echo "Built to ${<}, relocating to ${@}" - @mkdir -p target/artifacts/ - @cp -v \ - ${<} \ - ${@} +# Packaging targets that depend on VERSION live in Makefile.packaging to avoid +# running `cargo vdev version` when invoking non-packaging targets. .PHONY: package package: build ## Build the Vector archive ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) package archive -.PHONY: package-x86_64-unknown-linux-gnu-all -package-x86_64-unknown-linux-gnu-all: package-x86_64-unknown-linux-gnu package-deb-x86_64-unknown-linux-gnu package-rpm-x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm - -.PHONY: package-x86_64-unknown-linux-musl-all -package-x86_64-unknown-linux-musl-all: package-x86_64-unknown-linux-musl # .tar.gz - -.PHONY: package-aarch64-unknown-linux-musl-all -package-aarch64-unknown-linux-musl-all: package-aarch64-unknown-linux-musl # .tar.gz - -.PHONY: package-aarch64-unknown-linux-gnu-all -package-aarch64-unknown-linux-gnu-all: package-aarch64-unknown-linux-gnu package-deb-aarch64 package-rpm-aarch64 # .tar.gz, .deb, .rpm - -.PHONY: package-armv7-unknown-linux-gnueabihf-all -package-armv7-unknown-linux-gnueabihf-all: package-armv7-unknown-linux-gnueabihf package-deb-armv7-gnu package-rpm-armv7hl-gnu # .tar.gz, .deb, .rpm - -.PHONY: package-armv7-unknown-linux-musleabihf-all -package-armv7-unknown-linux-musleabihf-all: package-armv7-unknown-linux-musleabihf # .tar.gz - -.PHONY: package-arm-unknown-linux-gnueabi-all -package-arm-unknown-linux-gnueabi-all: package-arm-unknown-linux-gnueabi package-deb-arm-gnu # .tar.gz, .deb - -.PHONY: package-arm-unknown-linux-musleabi-all -package-arm-unknown-linux-musleabi-all: package-arm-unknown-linux-musleabi # .tar.gz - -.PHONY: package-x86_64-unknown-linux-gnu -package-x86_64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-gnu` triple. - @echo "Output to ${<}." - -.PHONY: package-x86_64-unknown-linux-musl -package-x86_64-unknown-linux-musl: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-musl` triple. - @echo "Output to ${<}." - -.PHONY: package-aarch64-unknown-linux-musl -package-aarch64-unknown-linux-musl: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-musl` triple. - @echo "Output to ${<}." - -.PHONY: package-aarch64-unknown-linux-gnu -package-aarch64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-gnu` triple. - @echo "Output to ${<}." - -.PHONY: package-armv7-unknown-linux-gnueabihf -package-armv7-unknown-linux-gnueabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-gnueabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-gnueabihf` triple. - @echo "Output to ${<}." - -.PHONY: package-armv7-unknown-linux-musleabihf -package-armv7-unknown-linux-musleabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-musleabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-musleabihf triple. - @echo "Output to ${<}." - -.PHONY: package-arm-unknown-linux-gnueabi -package-arm-unknown-linux-gnueabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-gnueabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-gnueabi` triple. - @echo "Output to ${<}." - -.PHONY: package-arm-unknown-linux-musleabi -package-arm-unknown-linux-musleabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-musleabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-musleabi` triple. - @echo "Output to ${<}." - -# debs - -.PHONY: package-deb-x86_64-unknown-linux-gnu -package-deb-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 GNU deb package - TARGET=x86_64-unknown-linux-gnu $(VDEV) package deb - -.PHONY: package-deb-x86_64-unknown-linux-musl -package-deb-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 GNU deb package - TARGET=x86_64-unknown-linux-musl $(VDEV) package deb - -.PHONY: package-deb-aarch64 -package-deb-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 deb package - TARGET=aarch64-unknown-linux-gnu $(VDEV) package deb - -.PHONY: package-deb-armv7-gnu -package-deb-armv7-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7-unknown-linux-gnueabihf deb package - TARGET=armv7-unknown-linux-gnueabihf $(VDEV) package deb - -.PHONY: package-deb-arm-gnu -package-deb-arm-gnu: package-arm-unknown-linux-gnueabi ## Build the arm-unknown-linux-gnueabi deb package - TARGET=arm-unknown-linux-gnueabi $(VDEV) package deb - -# rpms - -.PHONY: package-rpm-x86_64-unknown-linux-gnu -package-rpm-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 rpm package - TARGET=x86_64-unknown-linux-gnu $(VDEV) package rpm - -.PHONY: package-rpm-x86_64-unknown-linux-musl -package-rpm-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 musl rpm package - TARGET=x86_64-unknown-linux-musl $(VDEV) package rpm - -.PHONY: package-rpm-aarch64 -package-rpm-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 rpm package - TARGET=aarch64-unknown-linux-gnu $(VDEV) package rpm - -.PHONY: package-rpm-armv7hl-gnu -package-rpm-armv7hl-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7hl-unknown-linux-gnueabihf rpm package - TARGET=armv7-unknown-linux-gnueabihf ARCH=armv7hl $(VDEV) package rpm +package-%: + $(MAKE) -f Makefile.packaging $@ ##@ Releasing diff --git a/Makefile.packaging b/Makefile.packaging new file mode 100644 index 0000000000000..032e334094d19 --- /dev/null +++ b/Makefile.packaging @@ -0,0 +1,130 @@ +# Makefile.packaging +# +# Packaging and release targets for Vector. +# Separated from the main Makefile so that `cargo vdev version` (needed for +# VERSION) is only executed when packaging/release targets are invoked. +# +# Usage: +# make -f Makefile.packaging +# +# These targets are also accessible via the main Makefile, which forwards +# packaging and release targets here automatically. + +VDEV := cargo vdev + +# Set version — only evaluated when this Makefile is loaded +export VERSION ?= $(shell command -v cargo >/dev/null && $(VDEV) version || echo unknown) + +# Delegate cross-compilation prerequisites to the main Makefile +target/%/release/vector.tar.gz: + unset TRIPLE && $(MAKE) $@ + +##@ Packaging + +# archives +target/artifacts/vector-${VERSION}-%.tar.gz: export TRIPLE :=$(@:target/artifacts/vector-${VERSION}-%.tar.gz=%) +target/artifacts/vector-${VERSION}-%.tar.gz: override PROFILE =release +target/artifacts/vector-${VERSION}-%.tar.gz: target/%/release/vector.tar.gz + @echo "Built to ${<}, relocating to ${@}" + @mkdir -p target/artifacts/ + @cp -v \ + ${<} \ + ${@} + +.PHONY: package-x86_64-unknown-linux-gnu-all +package-x86_64-unknown-linux-gnu-all: package-x86_64-unknown-linux-gnu package-deb-x86_64-unknown-linux-gnu package-rpm-x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm + +.PHONY: package-x86_64-unknown-linux-musl-all +package-x86_64-unknown-linux-musl-all: package-x86_64-unknown-linux-musl # .tar.gz + +.PHONY: package-aarch64-unknown-linux-musl-all +package-aarch64-unknown-linux-musl-all: package-aarch64-unknown-linux-musl # .tar.gz + +.PHONY: package-aarch64-unknown-linux-gnu-all +package-aarch64-unknown-linux-gnu-all: package-aarch64-unknown-linux-gnu package-deb-aarch64 package-rpm-aarch64 # .tar.gz, .deb, .rpm + +.PHONY: package-armv7-unknown-linux-gnueabihf-all +package-armv7-unknown-linux-gnueabihf-all: package-armv7-unknown-linux-gnueabihf package-deb-armv7-gnu package-rpm-armv7hl-gnu # .tar.gz, .deb, .rpm + +.PHONY: package-armv7-unknown-linux-musleabihf-all +package-armv7-unknown-linux-musleabihf-all: package-armv7-unknown-linux-musleabihf # .tar.gz + +.PHONY: package-arm-unknown-linux-gnueabi-all +package-arm-unknown-linux-gnueabi-all: package-arm-unknown-linux-gnueabi package-deb-arm-gnu # .tar.gz, .deb + +.PHONY: package-arm-unknown-linux-musleabi-all +package-arm-unknown-linux-musleabi-all: package-arm-unknown-linux-musleabi # .tar.gz + +.PHONY: package-x86_64-unknown-linux-gnu +package-x86_64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-gnu` triple. + @echo "Output to ${<}." + +.PHONY: package-x86_64-unknown-linux-musl +package-x86_64-unknown-linux-musl: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-musl` triple. + @echo "Output to ${<}." + +.PHONY: package-aarch64-unknown-linux-musl +package-aarch64-unknown-linux-musl: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-musl` triple. + @echo "Output to ${<}." + +.PHONY: package-aarch64-unknown-linux-gnu +package-aarch64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-gnu` triple. + @echo "Output to ${<}." + +.PHONY: package-armv7-unknown-linux-gnueabihf +package-armv7-unknown-linux-gnueabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-gnueabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-gnueabihf` triple. + @echo "Output to ${<}." + +.PHONY: package-armv7-unknown-linux-musleabihf +package-armv7-unknown-linux-musleabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-musleabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-musleabihf triple. + @echo "Output to ${<}." + +.PHONY: package-arm-unknown-linux-gnueabi +package-arm-unknown-linux-gnueabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-gnueabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-gnueabi` triple. + @echo "Output to ${<}." + +.PHONY: package-arm-unknown-linux-musleabi +package-arm-unknown-linux-musleabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-musleabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-musleabi` triple. + @echo "Output to ${<}." + +# debs + +.PHONY: package-deb-x86_64-unknown-linux-gnu +package-deb-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 GNU deb package + TARGET=x86_64-unknown-linux-gnu $(VDEV) package deb + +.PHONY: package-deb-x86_64-unknown-linux-musl +package-deb-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 GNU deb package + TARGET=x86_64-unknown-linux-musl $(VDEV) package deb + +.PHONY: package-deb-aarch64 +package-deb-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 deb package + TARGET=aarch64-unknown-linux-gnu $(VDEV) package deb + +.PHONY: package-deb-armv7-gnu +package-deb-armv7-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7-unknown-linux-gnueabihf deb package + TARGET=armv7-unknown-linux-gnueabihf $(VDEV) package deb + +.PHONY: package-deb-arm-gnu +package-deb-arm-gnu: package-arm-unknown-linux-gnueabi ## Build the arm-unknown-linux-gnueabi deb package + TARGET=arm-unknown-linux-gnueabi $(VDEV) package deb + +# rpms + +.PHONY: package-rpm-x86_64-unknown-linux-gnu +package-rpm-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 rpm package + TARGET=x86_64-unknown-linux-gnu $(VDEV) package rpm + +.PHONY: package-rpm-x86_64-unknown-linux-musl +package-rpm-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 musl rpm package + TARGET=x86_64-unknown-linux-musl $(VDEV) package rpm + +.PHONY: package-rpm-aarch64 +package-rpm-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 rpm package + TARGET=aarch64-unknown-linux-gnu $(VDEV) package rpm + +.PHONY: package-rpm-armv7hl-gnu +package-rpm-armv7hl-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7hl-unknown-linux-gnueabihf rpm package + TARGET=armv7-unknown-linux-gnueabihf ARCH=armv7hl $(VDEV) package rpm + + From 35351b9cc9ae7dffdf1a461203137905f987a8b5 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 17 Apr 2026 17:22:12 -0400 Subject: [PATCH 092/364] chore(codecs): replace native encoding fixture patch files with cfg-gated code (#24971) * cfg-gate fixture generating code * Use generate-fixtures vrl branch * Move fixture generation into its own binary and use a set SEED * Gate from_size_and_seed behind generate-fixtures * Remove vrl_generate_fixtures.patch * Format * Fix clippy warnings * Fix markdown * Format again * fix(codecs): normalize -0.0 after rounding in fixture generation * chore(deps): update vrl dependency to main branch * Update VRL * fix(ci): enable vrl-functions-crypto for http-client integration tests --------- Co-authored-by: Pavlos Rontidis --- Cargo.lock | 19 +- Cargo.toml | 2 +- .../tests/data/native_encoding/README.md | 31 ++-- .../vector_generate_fixtures.patch | 163 ------------------ .../vrl_generate_fixtures.patch | 69 -------- lib/vector-core/Cargo.toml | 7 + lib/vector-core/src/bin/generate_fixtures.rs | 38 ++++ .../{test/common.rs => arbitrary_impl.rs} | 52 ++++-- lib/vector-core/src/event/metric/tags.rs | 4 +- lib/vector-core/src/event/mod.rs | 3 + lib/vector-core/src/event/test/mod.rs | 1 - .../src/event/test/serialization.rs | 5 +- lib/vector-core/src/event/test/size_of.rs | 3 +- lib/vector-core/src/metrics/ddsketch.rs | 11 ++ 14 files changed, 130 insertions(+), 278 deletions(-) delete mode 100644 lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch delete mode 100644 lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch create mode 100644 lib/vector-core/src/bin/generate_fixtures.rs rename lib/vector-core/src/event/{test/common.rs => arbitrary_impl.rs} (91%) diff --git a/Cargo.lock b/Cargo.lock index 2d641cc2a2934..3215e97f0fd44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,7 +1781,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2 1.0.106", "quote 1.0.44", "regex", @@ -8251,7 +8251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.4.1", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -8271,8 +8271,8 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", - "itertools 0.14.0", + "heck 0.5.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -8318,7 +8318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -10323,7 +10323,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -10335,7 +10335,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -12932,7 +12932,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vrl" version = "0.31.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#d21e53192e6a3ed3d38ef6f32886f1cbf30d308c" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#1357941d4481e84fb0323bd89c69cedf35c323f8" dependencies = [ "aes", "aes-siv", @@ -12966,6 +12966,7 @@ dependencies = [ "exitcode", "fancy-regex", "flate2", + "getrandom 0.3.4", "grok", "hex", "hmac", @@ -13005,7 +13006,7 @@ dependencies = [ "publicsuffix", "quickcheck", "quoted_printable", - "rand 0.8.5", + "rand 0.9.4", "regex", "relative-path 2.0.1", "reqwest 0.12.28", diff --git a/Cargo.toml b/Cargo.toml index 84ded822b5790..03aff35ec8c13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1031,7 +1031,7 @@ gcp-integration-tests = ["sinks-gcp"] gcp-pubsub-integration-tests = ["sinks-gcp", "sources-gcp_pubsub"] greptimedb-integration-tests = ["sinks-greptimedb_metrics", "sinks-greptimedb_logs"] humio-integration-tests = ["sinks-humio"] -http-client-integration-tests = ["sources-http_client"] +http-client-integration-tests = ["sources-http_client", "vrl-functions-crypto"] influxdb-integration-tests = ["sinks-influxdb"] kafka-integration-tests = ["sinks-kafka", "sources-kafka"] logstash-integration-tests = ["docker", "sources-logstash"] diff --git a/lib/codecs/tests/data/native_encoding/README.md b/lib/codecs/tests/data/native_encoding/README.md index 0a84c98043c41..2cf2aaba974be 100644 --- a/lib/codecs/tests/data/native_encoding/README.md +++ b/lib/codecs/tests/data/native_encoding/README.md @@ -6,28 +6,23 @@ and we test that all the examples can be successfully parsed, parse the same across both formats, and match the current serialized format. In order to avoid small inherent serialization differences between JSON and -protobuf (e.g. float handling), some changes were made to the `Arbitrary` -implementation for `Event` to give simpler values. These are not changes we want -in most property testing scenarios, but they are appropriate in this case where -we only care about the overall structure of the events. +protobuf (e.g. float handling), the `generate-fixtures` feature flag in +`vector-core` activates a stricter `Arbitrary` implementation for `Event` that +produces simpler, round-trip-safe f64 values and non-empty field names. These +changes are intentionally scoped to fixture generation and not used in regular +property testing. -There is currently a multi-step procedure to re-generate the data files. -There are two diffs committed to this directory: - - `vector_generate_fixtures.patch` - - `vrl_generate_fixtures.patch` +## Re-generating fixtures -The `vrl_` one must be applied to the vectordotdev/vrl repo. -The `vector_` one must be applied to the vector repo (you are here). +Both this repo and the VRL repo have a `generate-fixtures` feature flag that +activates fixture-stable `Arbitrary` implementations. The vector-core +`generate-fixtures` feature automatically enables `vrl/generate-fixtures`. -Part of the vector patch file is a `roundtrip` unit test definition that needs -to be evoked from `lib/vector-core`. Before invoking it, the `_json` and `_proto` -directories need to be created. +### Run the generator ```bash - $ cd lib/vector-core - $ mkdir _json/ proto/ - $ cargo test event::test::serialization::roundtrip +cargo run -p vector-core --features generate-fixtures --bin generate-fixtures ``` -That test case writes out the appropriate files into the dirs, which then need to be -moved to their location here. +The binary writes files directly into this directory's `json/` and `proto/` +subdirectories, replacing the existing fixtures. diff --git a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch deleted file mode 100644 index a0262928b3b3b..0000000000000 --- a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch +++ /dev/null @@ -1,163 +0,0 @@ -diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml -index 0f51fc830..aadf99841 100644 ---- a/lib/vector-core/Cargo.toml -+++ b/lib/vector-core/Cargo.toml -@@ -95,7 +95,7 @@ rand = "0.8.5" - rand_distr = "0.4.3" - tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } - vector-common = { path = "../vector-common", default-features = false, features = ["test"] } --vrl = { version = "0.9.0", features = ["cli", "test", "test_framework", "arbitrary"] } -+vrl = { path = "../../../vrl", features = ["cli", "test", "test_framework", "arbitrary"] } - - [features] - api = ["dep:async-graphql"] -diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/test/common.rs -index c7ccca952..f851288ce 100644 ---- a/lib/vector-core/src/event/test/common.rs -+++ b/lib/vector-core/src/event/test/common.rs -@@ -26,6 +26,15 @@ const ALPHABET: [&str; 27] = [ - "t", "u", "v", "w", "x", "y", "z", "_", - ]; - -+fn make_simple_f64(g: &mut Gen) -> f64 { -+ let mut value = f64::arbitrary(g) % MAX_F64_SIZE; -+ while value.is_nan() || value == -0.0 { -+ value = f64::arbitrary(g) % MAX_F64_SIZE; -+ } -+ value = (value * 10_000.0).round() / 10_000.0; -+ value -+} -+ - #[derive(Debug, Clone)] - pub struct Name { - inner: String, -@@ -34,7 +43,7 @@ pub struct Name { - impl Arbitrary for Name { - fn arbitrary(g: &mut Gen) -> Self { - let mut name = String::with_capacity(MAX_STR_SIZE); -- for _ in 0..(g.size() % MAX_STR_SIZE) { -+ for _ in 0..(usize::max(1, g.size() % MAX_STR_SIZE)) { - let idx: usize = usize::arbitrary(g) % ALPHABET.len(); - name.push_str(ALPHABET[idx]); - } -@@ -182,10 +191,10 @@ impl Arbitrary for MetricValue { - // here toward `MetricValue::Counter` and `MetricValue::Gauge`. - match u8::arbitrary(g) % 7 { - 0 => MetricValue::Counter { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - }, - 1 => MetricValue::Gauge { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - }, - 2 => MetricValue::Set { - values: BTreeSet::arbitrary(g), -@@ -197,19 +206,20 @@ impl Arbitrary for MetricValue { - 4 => MetricValue::AggregatedHistogram { - buckets: Vec::arbitrary(g), - count: u64::arbitrary(g), -- sum: f64::arbitrary(g) % MAX_F64_SIZE, -+ sum: make_simple_f64(g), - }, - 5 => MetricValue::AggregatedSummary { - quantiles: Vec::arbitrary(g), - count: u64::arbitrary(g), -- sum: f64::arbitrary(g) % MAX_F64_SIZE, -+ sum: make_simple_f64(g), - }, - 6 => { - // We're working around quickcheck's limitations here, and - // should really migrate the tests in question to use proptest - let num_samples = u8::arbitrary(g); - let samples = std::iter::repeat_with(|| loop { -- let f = f64::arbitrary(g); -+ // let f = f64::arbitrary(g); -+ let f = make_simple_f64(g); - if f.is_normal() { - return f; - } -@@ -219,6 +229,8 @@ impl Arbitrary for MetricValue { - - let mut sketch = AgentDDSketch::with_agent_defaults(); - sketch.insert_many(&samples); -+ sketch.sum = make_simple_f64(g); -+ sketch.avg = make_simple_f64(g); - - MetricValue::Sketch { - sketch: MetricSketch::AgentDDSketch(sketch), -@@ -368,7 +380,7 @@ impl Arbitrary for MetricValue { - impl Arbitrary for Sample { - fn arbitrary(g: &mut Gen) -> Self { - Sample { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - rate: u32::arbitrary(g), - } - } -@@ -398,8 +410,8 @@ impl Arbitrary for Sample { - impl Arbitrary for Quantile { - fn arbitrary(g: &mut Gen) -> Self { - Quantile { -- quantile: f64::arbitrary(g) % MAX_F64_SIZE, -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ quantile: make_simple_f64(g), -+ value: make_simple_f64(g), - } - } - -@@ -428,7 +440,7 @@ impl Arbitrary for Quantile { - impl Arbitrary for Bucket { - fn arbitrary(g: &mut Gen) -> Self { - Bucket { -- upper_limit: f64::arbitrary(g) % MAX_F64_SIZE, -+ upper_limit: make_simple_f64(g), - count: u64::arbitrary(g), - } - } -diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs -index aaab559da..5db6c0613 100644 ---- a/lib/vector-core/src/event/test/serialization.rs -+++ b/lib/vector-core/src/event/test/serialization.rs -@@ -96,3 +96,24 @@ fn type_serialization() { - assert_eq!(map["bool"], json!(true)); - assert_eq!(map["string"], json!("thisisastring")); - } -+ -+#[test] -+fn roundtrip() { -+ use prost::Message; -+ use quickcheck::{Arbitrary, Gen}; -+ use std::{fs::File, io::Write}; -+ -+ let mut gen = Gen::new(128); -+ for n in 0..1024 { -+ let mut json_out = File::create(format!("_json/{n:04}.json")).unwrap(); -+ let mut proto_out = File::create(format!("_proto/{n:04}.pb")).unwrap(); -+ let event = Event::arbitrary(&mut gen); -+ serde_json::to_writer(&mut json_out, &event).unwrap(); -+ -+ let array = EventArray::from(event); -+ let proto = proto::EventArray::from(array); -+ let mut buf = BytesMut::new(); -+ proto.encode(&mut buf).unwrap(); -+ proto_out.write_all(&buf).unwrap(); -+ } -+} -diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs -index 3d0f80bb5..fcc6bcc97 100644 ---- a/lib/vector-core/src/metrics/ddsketch.rs -+++ b/lib/vector-core/src/metrics/ddsketch.rs -@@ -229,10 +229,10 @@ pub struct AgentDDSketch { - max: f64, - - /// The sum of all observations within the sketch. -- sum: f64, -+ pub sum: f64, - - /// The average value of all observations within the sketch. -- avg: f64, -+ pub avg: f64, - } - - impl AgentDDSketch { diff --git a/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch deleted file mode 100644 index 430b74abb8b66..0000000000000 --- a/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch +++ /dev/null @@ -1,69 +0,0 @@ -diff --git a/src/lib.rs b/src/lib.rs -index 29cad8dfc..0b166beb2 100644 ---- a/src/lib.rs -+++ b/src/lib.rs -@@ -1,4 +1,4 @@ --#![deny(warnings)] -+// #![deny(warnings)] - #![deny(clippy::all)] - #![deny(unused_allocation)] - #![deny(unused_extern_crates)] -diff --git a/src/value/value/arbitrary.rs b/src/value/value/arbitrary.rs -index dde213281..33b2f2518 100644 ---- a/src/value/value/arbitrary.rs -+++ b/src/value/value/arbitrary.rs -@@ -1,7 +1,6 @@ - use std::collections::BTreeMap; - - use bytes::Bytes; --use chrono::{DateTime, NaiveDateTime, Utc}; - use ordered_float::NotNan; - use quickcheck::{Arbitrary, Gen}; - -@@ -11,16 +10,13 @@ const MAX_ARRAY_SIZE: usize = 4; - const MAX_MAP_SIZE: usize = 4; - const MAX_F64_SIZE: f64 = 1_000_000.0; - --fn datetime(g: &mut Gen) -> DateTime { -- // `chrono` documents that there is an out-of-range for both second and -- // nanosecond values but doesn't actually document what the valid ranges -- // are. We just sort of arbitrarily restrict things. -- let secs = i64::arbitrary(g) % 32_000; -- let nanosecs = u32::arbitrary(g) % 32_000; -- DateTime::::from_utc( -- NaiveDateTime::from_timestamp_opt(secs, nanosecs).expect("invalid timestamp"), -- Utc, -- ) -+fn make_simple_f64(g: &mut Gen) -> f64 { -+ let mut value = f64::arbitrary(g) % MAX_F64_SIZE; -+ while value.is_nan() || value == -0.0 { -+ value = f64::arbitrary(g) % MAX_F64_SIZE; -+ } -+ value = (value * 10_000.0).round() / 10_000.0; -+ value - } - - impl Arbitrary for Value { -@@ -32,18 +28,18 @@ impl Arbitrary for Value { - // field picking. - match u8::arbitrary(g) % 8 { - 0 => { -- let bytes: Vec = Vec::arbitrary(g); -+ let bytes = String::arbitrary(g); - Self::Bytes(Bytes::from(bytes)) - } - 1 => Self::Integer(i64::arbitrary(g)), - 2 => { -- let f = f64::arbitrary(g) % MAX_F64_SIZE; -+ //let f = f64::arbitrary(g) % MAX_F64_SIZE; -+ let f = make_simple_f64(g); - let not_nan = NotNan::new(f).unwrap_or_else(|_| NotNan::new(0.0).unwrap()); - Self::from(not_nan) - } - 3 => Self::Boolean(bool::arbitrary(g)), -- 4 => Self::Timestamp(datetime(g)), -- 5 => { -+ 4 | 5 => { - let mut gen = Gen::new(MAX_MAP_SIZE); - Self::Object(BTreeMap::arbitrary(&mut gen)) - } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 8a512884b2c32..7d2d8f8979266 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -65,6 +65,7 @@ vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vrl.workspace = true cfg-if.workspace = true +quickcheck = { workspace = true, optional = true } [target.'cfg(target_os = "macos")'.dependencies] security-framework = "3.6.0" @@ -100,6 +101,12 @@ default = [] lua = ["dep:mlua", "dep:tokio-stream", "vrl/lua"] vrl = [] test = ["vector-common/test", "proptest"] +generate-fixtures = ["vrl/generate-fixtures", "dep:quickcheck"] + +[[bin]] +name = "generate-fixtures" +path = "src/bin/generate_fixtures.rs" +required-features = ["generate-fixtures"] [[bench]] name = "event" diff --git a/lib/vector-core/src/bin/generate_fixtures.rs b/lib/vector-core/src/bin/generate_fixtures.rs new file mode 100644 index 0000000000000..9ab133757bbf8 --- /dev/null +++ b/lib/vector-core/src/bin/generate_fixtures.rs @@ -0,0 +1,38 @@ +use std::{fs::File, io::Write, path::PathBuf}; + +use bytes::BytesMut; +use prost::Message; +use quickcheck::{Arbitrary as _, Gen}; +use vector_core::event::{Event, EventArray, proto}; + +const SEED: u64 = 0; +const GEN_SIZE: usize = 128; + +fn main() { + let fixture_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../codecs/tests/data/native_encoding"); + let json_dir = fixture_dir.join("json"); + let proto_dir = fixture_dir.join("proto"); + std::fs::create_dir_all(&json_dir).unwrap(); + std::fs::create_dir_all(&proto_dir).unwrap(); + + let mut rng = Gen::from_size_and_seed(GEN_SIZE, SEED); + for n in 0..1024_usize { + let event = Event::arbitrary(&mut rng); + + let mut json_out = File::create(json_dir.join(format!("{n:04}.json"))).unwrap(); + serde_json::to_writer(&mut json_out, &event).unwrap(); + + let mut proto_out = File::create(proto_dir.join(format!("{n:04}.pb"))).unwrap(); + let mut buf = BytesMut::new(); + proto::EventArray::from(EventArray::from(event)) + .encode(&mut buf) + .unwrap(); + proto_out.write_all(&buf).unwrap(); + } + + #[allow(clippy::print_stdout)] + { + println!("Written 1024 fixtures to {}", fixture_dir.display()); + } +} diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/arbitrary_impl.rs similarity index 91% rename from lib/vector-core/src/event/test/common.rs rename to lib/vector-core/src/event/arbitrary_impl.rs index ab9460a7fa90f..fe02b846aa9b2 100644 --- a/lib/vector-core/src/event/test/common.rs +++ b/lib/vector-core/src/event/arbitrary_impl.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use quickcheck::{Arbitrary, Gen, empty_shrinker}; use vrl::value::{ObjectMap, Value}; -use super::super::{ +use super::{ Event, EventMetadata, LogEvent, Metric, MetricKind, MetricValue, StatisticKind, TraceEvent, metric::{ Bucket, MetricData, MetricName, MetricSeries, MetricSketch, MetricTags, MetricTime, @@ -21,6 +21,27 @@ const ALPHABET: [&str; 27] = [ "t", "u", "v", "w", "x", "y", "z", "_", ]; +// When generating fixtures we need f64 values that survive a JSON round-trip +// without any loss of precision or serialization ambiguity (NaN, -0.0). +// Under the `generate-fixtures` feature the helper produces clean values; +// otherwise it falls back to the standard quickcheck approach. +fn f64_for_arbitrary(g: &mut Gen) -> f64 { + #[cfg(feature = "generate-fixtures")] + { + let mut value = f64::arbitrary(g) % MAX_F64_SIZE; + while value.is_nan() || value == -0.0 { + value = f64::arbitrary(g) % MAX_F64_SIZE; + } + let rounded = (value * 10_000.0).round() / 10_000.0; + // Rounding can produce -0.0 from small negatives; normalize to +0.0. + if rounded == -0.0_f64 { 0.0 } else { rounded } + } + #[cfg(not(feature = "generate-fixtures"))] + { + f64::arbitrary(g) % MAX_F64_SIZE + } +} + #[derive(Debug, Clone)] pub struct Name { inner: String, @@ -29,7 +50,11 @@ pub struct Name { impl Arbitrary for Name { fn arbitrary(g: &mut Gen) -> Self { let mut name = String::with_capacity(MAX_STR_SIZE); - for _ in 0..(g.size() % MAX_STR_SIZE) { + #[cfg(feature = "generate-fixtures")] + let len = usize::max(1, g.size() % MAX_STR_SIZE); + #[cfg(not(feature = "generate-fixtures"))] + let len = g.size() % MAX_STR_SIZE; + for _ in 0..len { let idx: usize = usize::arbitrary(g) % ALPHABET.len(); name.push_str(ALPHABET[idx]); } @@ -76,6 +101,9 @@ impl Arbitrary for Event { impl Arbitrary for LogEvent { fn arbitrary(g: &mut Gen) -> Self { + #[cfg(feature = "generate-fixtures")] + let mut generator = Gen::from_size_and_seed(MAX_MAP_SIZE, u64::arbitrary(g)); + #[cfg(not(feature = "generate-fixtures"))] let mut generator = Gen::new(MAX_MAP_SIZE); let map: ObjectMap = ObjectMap::arbitrary(&mut generator); let metadata: EventMetadata = EventMetadata::arbitrary(g); @@ -175,10 +203,10 @@ impl Arbitrary for MetricValue { // here toward `MetricValue::Counter` and `MetricValue::Gauge`. match u8::arbitrary(g) % 7 { 0 => MetricValue::Counter { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), }, 1 => MetricValue::Gauge { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), }, 2 => MetricValue::Set { values: BTreeSet::arbitrary(g), @@ -190,12 +218,12 @@ impl Arbitrary for MetricValue { 4 => MetricValue::AggregatedHistogram { buckets: Vec::arbitrary(g), count: u64::arbitrary(g), - sum: f64::arbitrary(g) % MAX_F64_SIZE, + sum: f64_for_arbitrary(g), }, 5 => MetricValue::AggregatedSummary { quantiles: Vec::arbitrary(g), count: u64::arbitrary(g), - sum: f64::arbitrary(g) % MAX_F64_SIZE, + sum: f64_for_arbitrary(g), }, 6 => { // We're working around quickcheck's limitations here, and @@ -203,7 +231,7 @@ impl Arbitrary for MetricValue { let num_samples = u8::arbitrary(g); let samples = std::iter::repeat_with(|| { loop { - let f = f64::arbitrary(g); + let f = f64_for_arbitrary(g); if f.is_normal() { return f; } @@ -214,6 +242,8 @@ impl Arbitrary for MetricValue { let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_many(&samples); + #[cfg(feature = "generate-fixtures")] + sketch.set_sum_avg(f64_for_arbitrary(g), f64_for_arbitrary(g)); MetricValue::Sketch { sketch: MetricSketch::AgentDDSketch(sketch), @@ -363,7 +393,7 @@ impl Arbitrary for MetricValue { impl Arbitrary for Sample { fn arbitrary(g: &mut Gen) -> Self { Sample { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), rate: u32::arbitrary(g), } } @@ -393,8 +423,8 @@ impl Arbitrary for Sample { impl Arbitrary for Quantile { fn arbitrary(g: &mut Gen) -> Self { Quantile { - quantile: f64::arbitrary(g) % MAX_F64_SIZE, - value: f64::arbitrary(g) % MAX_F64_SIZE, + quantile: f64_for_arbitrary(g), + value: f64_for_arbitrary(g), } } @@ -423,7 +453,7 @@ impl Arbitrary for Quantile { impl Arbitrary for Bucket { fn arbitrary(g: &mut Gen) -> Self { Bucket { - upper_limit: f64::arbitrary(g) % MAX_F64_SIZE, + upper_limit: f64_for_arbitrary(g), count: u64::arbitrary(g), } } diff --git a/lib/vector-core/src/event/metric/tags.rs b/lib/vector-core/src/event/metric/tags.rs index 416f239af5ab5..52c9a365d2534 100644 --- a/lib/vector-core/src/event/metric/tags.rs +++ b/lib/vector-core/src/event/metric/tags.rs @@ -603,13 +603,13 @@ impl ByteSizeOf for MetricTags { } } -#[cfg(test)] +#[cfg(any(test, feature = "generate-fixtures"))] mod test_support { use std::collections::HashSet; use quickcheck::{Arbitrary, Gen}; - use super::*; + use super::{BTreeMap, MetricTags, TagValue, TagValueSet}; impl Arbitrary for TagValue { fn arbitrary(g: &mut Gen) -> Self { diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index 9d72301e85804..e70ebc4335c9d 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -23,6 +23,8 @@ pub use vrl_target::{TargetEvents, VrlTarget}; use crate::config::{LogNamespace, OutputId}; +#[cfg(any(test, feature = "generate-fixtures"))] +pub(crate) mod arbitrary_impl; pub mod array; pub mod discriminant; mod estimated_json_encoded_size_of; @@ -35,6 +37,7 @@ pub mod metric; pub mod proto; mod r#ref; mod ser; + #[cfg(test)] mod test; mod trace; diff --git a/lib/vector-core/src/event/test/mod.rs b/lib/vector-core/src/event/test/mod.rs index 3762a53363602..8f9cbad4da2c4 100644 --- a/lib/vector-core/src/event/test/mod.rs +++ b/lib/vector-core/src/event/test/mod.rs @@ -1,4 +1,3 @@ -mod common; mod serialization; mod size_of; diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs index 75304e3960d54..4bd7ac369e5c8 100644 --- a/lib/vector-core/src/event/test/serialization.rs +++ b/lib/vector-core/src/event/test/serialization.rs @@ -1,12 +1,11 @@ +use super::*; +use crate::config::log_schema; use bytes::{Buf, BufMut, BytesMut}; use quickcheck::{QuickCheck, TestResult}; use regex::Regex; use similar_asserts::assert_eq; use vector_buffers::encoding::Encodable; -use super::*; -use crate::config::log_schema; - fn encode_value(value: T, buffer: &mut B) { value.encode(buffer).expect("encoding should not fail"); } diff --git a/lib/vector-core/src/event/test/size_of.rs b/lib/vector-core/src/event/test/size_of.rs index 2428bc5ac6491..36bb1a0e23c4a 100644 --- a/lib/vector-core/src/event/test/size_of.rs +++ b/lib/vector-core/src/event/test/size_of.rs @@ -4,7 +4,8 @@ use lookup::{PathPrefix, path}; use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult}; use vector_common::byte_size_of::ByteSizeOf; -use super::{common::Name, *}; +use super::*; +use crate::event::arbitrary_impl::Name; #[test] fn at_least_wrapper_size() { diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index b66f25ceb150b..886a5c190bf32 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -284,6 +284,17 @@ impl AgentDDSketch { }) } + /// Overrides `sum` and `avg` with arbitrary values. + /// + /// Only available under the `generate-fixtures` feature, where we need to + /// produce sketches with independently-randomized summary statistics to + /// exercise round-trip serialization of those fields. + #[cfg(feature = "generate-fixtures")] + pub fn set_sum_avg(&mut self, sum: f64, avg: f64) { + self.sum = sum; + self.avg = avg; + } + pub fn gamma(&self) -> f64 { self.config.gamma_v } From 604866d560deea7a14cbfa767a8ca99eee07089a Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 20 Apr 2026 14:58:40 -0400 Subject: [PATCH 093/364] fix(website): use parent site for RUM to resolve intake host (#25224) --- website/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/config.toml b/website/config.toml index 414451d38b57d..15cab204233af 100644 --- a/website/config.toml +++ b/website/config.toml @@ -95,7 +95,7 @@ discord = "https://chat.vector.dev" [params.datadog_config] application_id = "894af722-5919-4a62-97fc-33f49bc52803" client_token = "pubcee1dc56e7f5bd03397cc3624eb7961c" -site = "cose.datadoghq.com" +site = "datadoghq.com" service_name = "vector" ## Menus From bdcf7476c55141ecf535cec6caafea7b2cc3d16d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:43:03 -0400 Subject: [PATCH 094/364] chore(ci): bump dorny/paths-filter from 3.0.2 to 4.0.1 (#25105) Bumps [dorny/paths-filter](https://github.com/dorny/paths-filter) from 3.0.2 to 4.0.1. - [Release notes](https://github.com/dorny/paths-filter/releases) - [Changelog](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md) - [Commits](https://github.com/dorny/paths-filter/compare/de90cc6fb38fc0963ad72b210f1f284cd68cea36...fbd0ab8f3e69293af611ebaee6363fc25e6d187d) --- updated-dependencies: - dependency-name: dorny/paths-filter dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/changes.yml | 6 +++--- .github/workflows/check_generated_vrl_docs.yml | 2 +- .github/workflows/integration_windows.yml | 2 +- .github/workflows/regression.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 999548111e9a7..f9cd9ce4ba5b9 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -189,7 +189,7 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ env.BASE_SHA }} @@ -360,7 +360,7 @@ jobs: - name: Create filter rules for integrations run: vdev int ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ env.BASE_SHA }} @@ -454,7 +454,7 @@ jobs: - name: Create filter rules for e2e tests run: vdev e2e ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ env.BASE_SHA }} diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index c2a9852474461..867ffd7619379 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -28,7 +28,7 @@ jobs: docs: ${{ steps.filter.outputs.docs }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: filters: | diff --git a/.github/workflows/integration_windows.yml b/.github/workflows/integration_windows.yml index 39289d786fd24..1074ba4a3b97b 100644 --- a/.github/workflows/integration_windows.yml +++ b/.github/workflows/integration_windows.yml @@ -16,7 +16,7 @@ jobs: windows: ${{ steps.filter.outputs.windows }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: filters: | diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index cc1170a8aaa21..aaf71eb9687a4 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -124,7 +124,7 @@ jobs: - name: Collect file changes id: changes - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 with: base: ${{ needs.resolve-inputs.outputs.baseline-sha }} ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} From c2022541ee792988c465ca9a76b93a0f54ea4e00 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 20 Apr 2026 15:43:44 -0400 Subject: [PATCH 095/364] docs(deprecations): add greptimedb v0 support deprecation entry (#25226) --- docs/DEPRECATIONS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index cf3a9377bfe24..b57db75f736af 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -21,3 +21,5 @@ For example: ## To be migrated ## To be removed + +- `v0.56.0` | `greptimedb-v0-support` | The `greptimedb_metrics` and `greptimedb_logs` sinks drop support for GreptimeDB v0.x. Users must upgrade their GreptimeDB instance to v1.x before upgrading Vector. From 0f7574ad33ce07243ab8f9d6399c5e6c2f451af1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 20 Apr 2026 16:49:34 -0400 Subject: [PATCH 096/364] revert(observability): fix panic in allocation tracing dealloc path (#25222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "fix(observability): fix panic in allocation tracing dealloc path (#25…" This reverts commit 43f620435a353b048c40661f59a2d2d18f2419ed. --- ...ix_allocation_tracing_dealloc_panic.fix.md | 3 - .../allocator/tracing_allocator.rs | 202 ++---------------- 2 files changed, 13 insertions(+), 192 deletions(-) delete mode 100644 changelog.d/fix_allocation_tracing_dealloc_panic.fix.md diff --git a/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md b/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md deleted file mode 100644 index 732c836eac0e8..0000000000000 --- a/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a panic (abort) when running with `--allocation-tracing` in debug builds, caused by deallocating memory that was allocated before tracking was enabled. Also fixed per-group memory accounting skew for reentrant allocations whose tracing closure was skipped, which left the group ID header uninitialized and caused deallocations to be attributed to wrong groups. - -authors: pront diff --git a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs index d0a2027eeb2e4..8d1eff3e36d4e 100644 --- a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs +++ b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs @@ -9,24 +9,8 @@ use super::{ }; use crate::internal_telemetry::allocations::TRACK_ALLOCATIONS; -/// Header value for allocations made while tracking is disabled. -/// On deallocation we free with the wrapped layout but skip tracing. -const UNTRACKED: u8 = 0; - -/// Header value for allocations whose tracing closure was skipped due to -/// reentrancy. `register()` never hands out `u8::MAX`, so this cannot -/// collide with a real group ID. On deallocation we free with the wrapped -/// layout but skip `trace_deallocation` to keep accounting balanced. -const UNTRACED: u8 = u8::MAX; - /// A tracing allocator that groups allocation events by groups. /// -/// Every allocation made through this allocator uses the "wrapped" layout: -/// the requested layout extended by one byte to store an allocation group -/// ID. This byte is always present regardless of whether tracking is -/// enabled, which guarantees that `dealloc` can always read a valid header -/// and free with the correct (wrapped) layout. -/// /// This allocator can only be used when specified via `#[global_allocator]`. pub struct GroupedTraceableAllocator { allocator: A, @@ -45,6 +29,11 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn alloc(&self, object_layout: Layout) -> *mut u8 { unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + return self.allocator.alloc(object_layout); + } + + // Allocate our wrapped layout and make sure the allocation succeeded. let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); let actual_ptr = self.allocator.alloc(actual_layout); if actual_ptr.is_null() { @@ -53,16 +42,6 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - group_id_ptr.write(UNTRACKED); - return actual_ptr; - } - - // Write the untraced sentinel so that `dealloc` always finds a - // valid header, even when the tracing closure below is skipped - // due to reentrancy. - group_id_ptr.write(UNTRACED); - let object_size = object_layout.size(); try_with_suspended_allocation_group( @@ -79,19 +58,19 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn dealloc(&self, object_ptr: *mut u8, object_layout: Layout) { unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + self.allocator.dealloc(object_ptr, object_layout); + return; + } + // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the + // requested layout, not the wrapped layout that was actually allocated. let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); + let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); - // Always free with the wrapped layout since all allocations - // (tracked or not) use it. + // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. self.allocator.dealloc(object_ptr, wrapped_layout); - // Skip tracing for untracked (tracking was off) and untraced - // (reentrant, closure skipped) allocations. - if raw_group_id == UNTRACKED || raw_group_id == UNTRACED { - return; - } - let object_size = object_layout.size(); let source_group_id = AllocationGroupId::from_raw(raw_group_id); @@ -117,158 +96,3 @@ fn get_wrapped_layout(object_layout: Layout) -> (Layout, usize) { (actual_layout.pad_to_align(), offset_to_group_id) } - -#[cfg(test)] -mod tests { - use std::{ - alloc::{GlobalAlloc, Layout, System}, - sync::atomic::{AtomicUsize, Ordering}, - }; - - use serial_test::serial; - - use super::*; - use crate::internal_telemetry::allocations::allocator::{ - token::AllocationGroupId, tracer::Tracer, - }; - - /// RAII guard that enables `TRACK_ALLOCATIONS` on creation and - /// restores it to `false` on drop, ensuring cleanup even if the - /// test panics. - struct TrackingGuard; - - impl TrackingGuard { - fn enable() -> Self { - TRACK_ALLOCATIONS.store(true, Ordering::Release); - Self - } - } - - impl Drop for TrackingGuard { - fn drop(&mut self) { - TRACK_ALLOCATIONS.store(false, Ordering::Release); - } - } - - struct CountingTracer { - allocs: AtomicUsize, - deallocs: AtomicUsize, - } - - impl CountingTracer { - const fn new() -> Self { - Self { - allocs: AtomicUsize::new(0), - deallocs: AtomicUsize::new(0), - } - } - } - - impl Tracer for CountingTracer { - fn trace_allocation(&self, _size: usize, _group_id: AllocationGroupId) { - self.allocs.fetch_add(1, Ordering::Relaxed); - } - - fn trace_deallocation(&self, _size: usize, _source_group_id: AllocationGroupId) { - self.deallocs.fetch_add(1, Ordering::Relaxed); - } - } - - #[test] - fn sentinels_do_not_collide_with_root_id() { - assert_eq!(UNTRACED, u8::MAX); - assert_ne!(UNTRACED, AllocationGroupId::ROOT.as_raw()); - assert_ne!(UNTRACKED, AllocationGroupId::ROOT.as_raw()); - } - - /// Allocations made while tracking is disabled get UNTRACKED (0) in the - /// header. Deallocating them (whether tracking is on or off) must use - /// the wrapped layout and skip tracing. - #[test] - #[serial] - fn untracked_alloc_dealloc_skips_tracing() { - let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); - let layout = Layout::from_size_align(64, 8).unwrap(); - - // Tracking starts off (default state). Allocate while disabled. - let ptr = unsafe { allocator.alloc(layout) }; - assert!(!ptr.is_null()); - - // Header must be UNTRACKED. - let (_, offset) = get_wrapped_layout(layout); - let raw_id = unsafe { ptr.add(offset).cast::().read() }; - assert_eq!(raw_id, UNTRACKED); - - // Enable tracking, then dealloc -- must not panic, no trace events. - let _guard = TrackingGuard::enable(); - unsafe { allocator.dealloc(ptr, layout) }; - - assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); - assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); - } - - /// Tracked allocation: header is a valid non-zero, non-sentinel group - /// ID, tracing fires for both alloc and dealloc, and dealloc completes - /// without panic. - #[test] - #[serial] - fn tracked_alloc_dealloc_does_not_panic() { - let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); - let layout = Layout::from_size_align(64, 8).unwrap(); - - let _guard = TrackingGuard::enable(); - let ptr = unsafe { allocator.alloc(layout) }; - assert!(!ptr.is_null()); - - let (_, offset) = get_wrapped_layout(layout); - let raw_id = unsafe { ptr.add(offset).cast::().read() }; - assert_eq!( - raw_id, - AllocationGroupId::ROOT.as_raw(), - "header must be the ROOT group ID" - ); - assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 1); - - unsafe { allocator.dealloc(ptr, layout) }; - - assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 1); - } - - /// End-to-end reentrant allocation: hold a mutable borrow on the - /// thread-local group stack so `try_with_suspended_allocation_group` - /// skips the tracing closure. The header must be UNTRACED and both - /// trace counters must stay at zero through alloc + dealloc. - #[test] - #[serial] - fn reentrant_alloc_writes_untraced_and_skips_tracing() { - use crate::internal_telemetry::allocations::allocator::token::LOCAL_ALLOCATION_GROUP_STACK; - - let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); - let layout = Layout::from_size_align(64, 8).unwrap(); - - let _guard = TrackingGuard::enable(); - - // Hold a mutable borrow to simulate reentrancy -- any nested - // `try_borrow_mut` inside `try_with_suspended_allocation_group` - // will fail, causing the tracing closure to be skipped. - LOCAL_ALLOCATION_GROUP_STACK.with(|group_stack| { - let _borrow = group_stack.borrow_mut(); - - let ptr = unsafe { allocator.alloc(layout) }; - assert!(!ptr.is_null()); - - let (_, offset) = get_wrapped_layout(layout); - let raw_id = unsafe { ptr.add(offset).cast::().read() }; - assert_eq!( - raw_id, UNTRACED, - "reentrant alloc must write UNTRACED sentinel" - ); - - assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); - - unsafe { allocator.dealloc(ptr, layout) }; - - assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); - }); - } -} From 3ca3e61ec8b8cd89be836b47346343069b229398 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 20 Apr 2026 17:11:31 -0400 Subject: [PATCH 097/364] fix(ci): use CUE raw multi-line strings in release generator (#25228) Fragment descriptions are embedded into CUE via `"""..."""`, where backslashes are still interpreted as escape sequences. A fragment containing e.g. a shell line continuation (`\`) produces invalid CUE and breaks `cue export`, which is then used by the next `cargo vdev release prepare` invocation. Switch to raw multi-line strings (`#"""..."""#`) so backslashes are literal and no escaping is needed on the description content. Co-authored-by: Claude Opus 4.7 (1M context) --- scripts/generate-release-cue.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate-release-cue.rb b/scripts/generate-release-cue.rb index 80a35c0683015..d11d9f8f02c8a 100755 --- a/scripts/generate-release-cue.rb +++ b/scripts/generate-release-cue.rb @@ -217,11 +217,14 @@ def generate_changelog!(new_version) # Note: `pr_numbers`, `scopes` and `breaking` are being omitted from the entries. # These are currently not required for rendering in the website. + # Use CUE raw multi-line strings (#"""..."""#) so backslashes in the + # fragment (e.g. shell line continuations) are not interpreted as + # escape sequences. entry = "{\n" + "type: #{type.to_json}\n" + - "description: \"\"\"\n" + + "description: #\"\"\"\n" + "#{description}\n" + - "\"\"\"\n" + "\"\"\"#\n" if contributors.length() > 0 entry += "contributors: #{contributors.to_json}\n" From 1c70988b54156abf8d031538f0f81f28e7c0a0e4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 21 Apr 2026 13:00:33 -0400 Subject: [PATCH 098/364] fix(api): restore HTTP GET /health endpoint (#25234) * refactor(api): serve gRPC via hyper + axum router Convert tonic's Server to an axum Router via into_router(), then serve over the same TcpListener via hyper::Server. Enables HTTP/1.1 acceptance so additional HTTP routes can be added alongside gRPC on the same port. Behavior-preserving. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(api): restore HTTP GET /health endpoint Re-expose the HTTP health endpoint that was removed as part of the GraphQL-to-gRPC migration (#24364). The endpoint matches the pre-0.55 response shape: 200 with body {"ok": true} while serving and 503 with body {"ok": false} after set_not_serving() is called during drain. HEAD is also handled. gRPC clients continue to use grpc.health.v1.Health/Check; both probes now share the same serving state so they agree during shutdown. Co-Authored-By: Claude Opus 4.7 (1M context) * test(api): cover HTTP GET/HEAD /health endpoint Adds two integration tests hitting the restored HTTP health endpoint via reqwest: - GET returns 200 with body {"ok":true} - HEAD returns 200 Exposes harness.api_port() so tests can reach the API port directly. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(api): document HTTP /health endpoint Document the HTTP GET/HEAD /health endpoint served alongside the gRPC API, framed as compatibility with Vector 0.54.0 and earlier. Updates the reference endpoints schema to allow HEAD, adds HEAD/GET entries for /health in api.cue with 200/503 responses, and adds a curl example to the API reference page. Co-Authored-By: Claude Opus 4.7 (1M context) * style(api): apply cargo fmt Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- src/api/grpc_server.rs | 85 +++++++++++++++++++++--- tests/vector_api/harness.rs | 5 ++ tests/vector_api/health.rs | 44 +++++++++++- website/content/en/docs/reference/api.md | 13 ++++ website/cue/reference.cue | 1 + website/cue/reference/api.cue | 25 ++++++- 6 files changed, 162 insertions(+), 11 deletions(-) diff --git a/src/api/grpc_server.rs b/src/api/grpc_server.rs index ce33b0da3f7d5..715aeecad2b8e 100644 --- a/src/api/grpc_server.rs +++ b/src/api/grpc_server.rs @@ -1,6 +1,20 @@ -use std::{error::Error as StdError, net::SocketAddr}; +use std::{ + error::Error as StdError, + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use axum::{ + Router, + extract::State, + http::{StatusCode, header}, + response::IntoResponse, + routing::get, +}; use tokio::sync::oneshot; -use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server as TonicServer; use tonic_health::server::{HealthReporter, health_reporter}; use vector_lib::tap::topology::WatchRx; @@ -8,10 +22,15 @@ use vector_lib::tap::topology::WatchRx; use super::grpc::ObservabilityService; use crate::{config::Config, proto::observability::Server as ObservabilityServer}; +/// Shared flag backing the HTTP `/health` endpoint. Mirrors the gRPC +/// `HealthReporter` serving status so HTTP and gRPC probes agree. +type ServingState = Arc; + /// gRPC API server for Vector observability. pub struct GrpcServer { _shutdown: oneshot::Sender<()>, health_reporter: HealthReporter, + serving: ServingState, addr: SocketAddr, } @@ -46,12 +65,22 @@ impl GrpcServer { // The empty service ("") is registered as SERVING by default. let (health_reporter, health_service) = health_reporter(); + let serving: ServingState = Arc::new(AtomicBool::new(true)); + let (_shutdown, rx) = oneshot::channel(); + // Convert the tokio TcpListener into a std listener for hyper's Server. + let std_listener = listener + .into_std() + .map_err(|e| crate::Error::from(format!("Failed to convert TCP listener: {}", e)))?; + std_listener.set_nonblocking(true).map_err(|e| { + crate::Error::from(format!("Failed to set TCP listener non-blocking: {}", e)) + })?; + + let router_serving = Arc::clone(&serving); + // Spawn the server with the already-bound listener tokio::spawn(async move { - let incoming = TcpListenerStream::new(listener); - // Build reflection service for tools like grpcurl let reflection_service = tonic_reflection::server::Builder::configure() .register_encoded_file_descriptor_set( @@ -61,11 +90,21 @@ impl GrpcServer { .build() .expect("Failed to build reflection service"); - let result = TonicServer::builder() + // Build the tonic router (gRPC services) and merge with the HTTP router + // so both protocols share the same port. `accept_http1(true)` lets plain + // HTTP/1.1 requests reach the merged axum routes. + let router = TonicServer::builder() + .accept_http1(true) .add_service(health_service) .add_service(ObservabilityServer::new(service)) .add_service(reflection_service) - .serve_with_incoming_shutdown(incoming, async { + .into_router() + .merge(http_router(router_serving)); + + let result = hyper::Server::from_tcp(std_listener) + .expect("Failed to build HTTP server from TCP listener") + .serve(router.into_make_service()) + .with_graceful_shutdown(async { rx.await.ok(); info!("GRPC API server shutting down."); }) @@ -86,6 +125,7 @@ impl GrpcServer { Ok(Self { _shutdown, health_reporter, + serving, addr: actual_addr, }) } @@ -93,9 +133,10 @@ impl GrpcServer { /// Signal that the server is no longer serving. /// /// Call this **before** draining the topology so that Kubernetes gRPC - /// readiness probes fail early and the pod is removed from endpoints - /// before the process exits. + /// readiness probes and HTTP `/health` probes fail early and the pod is + /// removed from endpoints before the process exits. pub async fn set_not_serving(&mut self) { + self.serving.store(false, Ordering::Relaxed); self.health_reporter .set_service_status("", tonic_health::ServingStatus::NotServing) .await; @@ -106,3 +147,31 @@ impl GrpcServer { self.addr } } + +/// Axum router exposing `GET`/`HEAD /health`. +/// +/// Returns `200 {"ok":true}` while the server is serving and +/// `503 {"ok":false}` once [`GrpcServer::set_not_serving`] has been called. +/// Matches the response shape of the pre-gRPC GraphQL-era endpoint so +/// existing HTTP health probes (Kubernetes, load balancers) keep working. +fn http_router(state: ServingState) -> Router { + Router::new() + .route("/health", get(health_handler).head(health_handler)) + .with_state(state) +} + +async fn health_handler(State(state): State) -> impl IntoResponse { + if state.load(Ordering::Relaxed) { + ( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + r#"{"ok":true}"#, + ) + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::CONTENT_TYPE, "application/json")], + r#"{"ok":false}"#, + ) + } +} diff --git a/tests/vector_api/harness.rs b/tests/vector_api/harness.rs index 274592fac5dae..ba79b521bba22 100644 --- a/tests/vector_api/harness.rs +++ b/tests/vector_api/harness.rs @@ -133,6 +133,11 @@ impl TestHarness { &mut self.api_client } + /// Returns the TCP port the API server is bound to + pub fn api_port(&self) -> u16 { + self.api_port + } + /// Reloads Vector configuration by sending SIGHUP or using watch mode /// /// Polls Vector to detect crashes early and succeed fast when reload completes. diff --git a/tests/vector_api/health.rs b/tests/vector_api/health.rs index a17d16eecf913..1e88c5aa34ed0 100644 --- a/tests/vector_api/health.rs +++ b/tests/vector_api/health.rs @@ -1,4 +1,7 @@ -//! Integration tests for the standard gRPC health check on the observability API. +//! Integration tests for the health endpoints on the observability API: +//! +//! * the standard gRPC health check (`grpc.health.v1.Health/Check`) +//! * the HTTP `/health` endpoint served on the same port use super::{common::*, harness::*}; @@ -23,3 +26,42 @@ async fn health_check_reports_serving() { assert!(harness.check_running(), "Vector should still be running"); } + +/// Verifies the HTTP `GET /health` endpoint returns 200 with `{"ok":true}` on a +/// running Vector instance. This endpoint is load-balancer friendly and is +/// shared with gRPC clients on the same API port. +#[tokio::test] +async fn http_health_endpoint_returns_200_when_serving() { + let config = single_source_config("demo", 1.0, None); + let harness = TestHarness::new(&config) + .await + .expect("Vector should start"); + + let url = format!("http://127.0.0.1:{}/health", harness.api_port()); + let response = reqwest::get(&url) + .await + .expect("GET /health should succeed"); + + assert_eq!(response.status(), reqwest::StatusCode::OK); + let body = response.text().await.expect("body should be readable"); + assert_eq!(body, r#"{"ok":true}"#); +} + +/// Verifies HTTP `HEAD /health` returns 200 without a body. ALB/ELB style +/// probes that prefer HEAD should work the same as GET. +#[tokio::test] +async fn http_health_endpoint_supports_head() { + let config = single_source_config("demo", 1.0, None); + let harness = TestHarness::new(&config) + .await + .expect("Vector should start"); + + let url = format!("http://127.0.0.1:{}/health", harness.api_port()); + let response = reqwest::Client::new() + .head(&url) + .send() + .await + .expect("HEAD /health should succeed"); + + assert_eq!(response.status(), reqwest::StatusCode::OK); +} diff --git a/website/content/en/docs/reference/api.md b/website/content/en/docs/reference/api.md index 949353241929b..9d49b863eb8ab 100644 --- a/website/content/en/docs/reference/api.md +++ b/website/content/en/docs/reference/api.md @@ -21,6 +21,12 @@ instance. This page covers how to configure and enable Vector's API. The API exposes a gRPC service defined in [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). You can interact with it using any standard gRPC tooling. +For compatibility with Vector 0.54.0 and earlier, the HTTP `GET /health` +endpoint continues to be served on the same port as the gRPC API, so +existing HTTP probes (for example AWS ALB health checks and Kubernetes +HTTP liveness/readiness probes) keep working without changes. See the +[Endpoints](#endpoints) section above for details. + ### Example using grpcurl ```bash @@ -35,3 +41,10 @@ grpcurl -plaintext \ -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents ``` + +### Example using curl (HTTP health) + +```bash +# 200 with body {"ok":true} while serving, 503 {"ok":false} during drain +curl -i http://localhost:8686/health +``` diff --git a/website/cue/reference.cue b/website/cue/reference.cue index 05d22f27e1f1a..789e2e66bce82 100644 --- a/website/cue/reference.cue +++ b/website/cue/reference.cue @@ -68,6 +68,7 @@ _values: { #Endpoints: [Path=string]: { DELETE?: #Endpoint GET?: #Endpoint + HEAD?: #Endpoint POST?: #Endpoint PUT?: #Endpoint } diff --git a/website/cue/reference/api.cue b/website/cue/reference/api.cue index 38feeff331893..bb1e3baf129ba 100644 --- a/website/cue/reference/api.cue +++ b/website/cue/reference/api.cue @@ -24,13 +24,34 @@ api: { "/health": { GET: { description: """ - Healthcheck endpoint. Useful to verify that - Vector is up and running. + HTTP healthcheck endpoint served on the same port as the + gRPC API, preserved for compatibility with Vector 0.54.0 + and earlier so existing HTTP probes (for example AWS ALB + and Kubernetes HTTP probes) keep working unchanged. + The response body is `{"ok": true}` while Vector is + serving and `{"ok": false}` once Vector begins draining. """ responses: { "200": { description: "Vector is initialized and running." } + "503": { + description: "Vector is draining or shutting down and should be removed from the load balancer." + } + } + } + HEAD: { + description: """ + Same semantics as `GET /health` but returns no body. + Intended for load balancer probes that prefer `HEAD`. + """ + responses: { + "200": { + description: "Vector is initialized and running." + } + "503": { + description: "Vector is draining or shutting down and should be removed from the load balancer." + } } } } From aafd4cb44f5649e692722b97d82973fea5509a41 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 21 Apr 2026 14:21:39 -0400 Subject: [PATCH 099/364] fix(demo_logs source): drop fakedata_generator, fix broken fake domains (#25236) * fix(demo_logs source): drop fakedata_generator, fix broken fake domains The fakedata_generator crate (v0.7.1) has an upstream bug: its get_dataset() match table is missing a "tlds" arm, so gen_domain() falls through to an empty JSON string. That fails to parse and the crate prints "Failed getting dataset for tlds. EOF while parsing a value at line 1 column 0" to stderr on every call, while gen_domain() returns ".Error: dataset not found" as the fake domain. Drop the dependency (and its transitive passt dep) and inline the three functions demo_logs actually used: gen_domain, gen_ipv4, gen_username. Domains now look like "random.io" instead of "random.Error: dataset not found", and stderr is quiet. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): changelog fragment for demo_logs fakedata fix Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): drop stale comment, allow TLDs/fakedata in spelling Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): swap fake domain name list to unique entries Replace the 8-word domain-name pool with fresh placeholder-style entries so nothing in lib/fakedata derives from the fakedata_generator crate's data. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/actions/spelling/allow.txt | 2 + Cargo.lock | 19 -------- LICENSE-3rdparty.csv | 2 - changelog.d/drop_fakedata_generator.fix.md | 9 ++++ lib/fakedata/Cargo.toml | 1 - lib/fakedata/src/logs.rs | 54 ++++++++++++++++++++-- 6 files changed, 61 insertions(+), 26 deletions(-) create mode 100644 changelog.d/drop_fakedata_generator.fix.md diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index b43abdd6fba66..db6d30301dcd7 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -154,6 +154,7 @@ Evercoss exactlyonce Explay Fabro +fakedata fakeintake FAQs fargate @@ -509,6 +510,7 @@ timeseries timespan timestamped Tizen +TLDs Tmobile tobz Tomtec diff --git a/Cargo.lock b/Cargo.lock index 3215e97f0fd44..2a7c05bd91410 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3909,22 +3909,9 @@ name = "fakedata" version = "0.1.0" dependencies = [ "chrono", - "fakedata_generator", "rand 0.9.4", ] -[[package]] -name = "fakedata_generator" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42ec18312c068f0d6008ad66ad2fa289ab40a16d07a06406c200baa09eb1e7c" -dependencies = [ - "passt", - "rand 0.9.4", - "serde", - "serde_json", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -7615,12 +7602,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" -[[package]] -name = "passt" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13242a5ce97f39a8095d03c8b273e91d09f2690c0b7d69a2af844941115bab24" - [[package]] name = "paste" version = "1.0.15" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ed18e3c3c172b..eb9e179e57c2b 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -270,7 +270,6 @@ event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apach evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset evmap-derive,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber -fakedata_generator,https://github.com/KevinGimbel/fakedata_generator,MIT,Kevin Gimbel fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina @@ -536,7 +535,6 @@ parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'A parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow parse-size,https://github.com/kennytm/parse-size,MIT,kennytm -passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers diff --git a/changelog.d/drop_fakedata_generator.fix.md b/changelog.d/drop_fakedata_generator.fix.md new file mode 100644 index 0000000000000..500e826bc3239 --- /dev/null +++ b/changelog.d/drop_fakedata_generator.fix.md @@ -0,0 +1,9 @@ +The text content generated by the `demo_logs` source has changed: the +pool of fake usernames and the pool of fake domain TLDs are now both +defined inside Vector rather than pulled from an external crate. The +line formats (`apache_common`, `apache_error`, `json`, `syslog`, +`bsd_syslog`) are unchanged. If any of your tests or downstream +pipelines assert on specific generated usernames or TLDs, please +update those expectations. + +authors: pront diff --git a/lib/fakedata/Cargo.toml b/lib/fakedata/Cargo.toml index 576ad22e96cad..52ba22387ef89 100644 --- a/lib/fakedata/Cargo.toml +++ b/lib/fakedata/Cargo.toml @@ -8,5 +8,4 @@ license = "MPL-2.0" [dependencies] chrono.workspace = true -fakedata_generator = "0.7.1" rand.workspace = true diff --git a/lib/fakedata/src/logs.rs b/lib/fakedata/src/logs.rs index 1c3aa7683d2c4..768d67b06890d 100644 --- a/lib/fakedata/src/logs.rs +++ b/lib/fakedata/src/logs.rs @@ -3,9 +3,44 @@ use chrono::{ format::{DelayedFormat, StrftimeItems}, prelude::Local, }; -use fakedata_generator::{gen_domain, gen_ipv4, gen_username}; use rand::{Rng, rng}; +static FAKE_USERNAMES: [&str; 20] = [ + "log_whisperer", + "commit_conductor", + "cache_cowboy", + "compile_captain", + "latency_llama", + "yaml_yoda", + "regex_rider", + "semver_sage", + "kernel_keith", + "pixel_pilgrim", + "kubectl_kev", + "pipeline_pat", + "telemetry_tina", + "merge_maria", + "parser_pete", + "debug_duchess", + "nullable_nate", + "grep_greg", + "stderr_stan", + "segfault_sue", +]; + +static FAKE_DOMAIN_NAMES: [&str; 8] = [ + "acme", + "contoso", + "widgets", + "example", + "placeholder", + "sample", + "foobar", + "testbench", +]; + +static FAKE_DOMAIN_TLDS: [&str; 8] = ["com", "net", "org", "io", "dev", "co", "app", "biz"]; + static APPLICATION_NAMES: [&str; 10] = [ "auth", "data", "deploy", "etl", "scraper", "cron", "ingress", "egress", "alerter", "fwd", ]; @@ -156,7 +191,11 @@ fn application() -> &'static str { } fn domain() -> String { - gen_domain() + format!( + "{}.{}", + random_from_array(&FAKE_DOMAIN_NAMES), + random_from_array(&FAKE_DOMAIN_TLDS), + ) } fn error_level() -> &'static str { @@ -188,7 +227,14 @@ fn http_version() -> &'static str { } fn ipv4_address() -> String { - gen_ipv4() + let mut r = rng(); + format!( + "{}.{}.{}.{}", + r.random_range(1..255), + r.random_range(1..255), + r.random_range(1..255), + r.random_range(1..255), + ) } fn pid() -> usize { @@ -208,7 +254,7 @@ fn referer() -> String { } fn username() -> String { - gen_username() + (*random_from_array(&FAKE_USERNAMES)).to_string() } fn syslog_version() -> usize { From 8cf773df0e3b219e40537119aa2b95b916342df2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Apr 2026 14:31:21 -0400 Subject: [PATCH 100/364] chore(releasing): start 0.56.0 development cycle (#25242) * chore(releasing): prepare v0.55.0 release (#25229) * chore(releasing): Pinned VRL version to 0.32.0 * chore(releasing): Generated release CUE file * chore(releasing): Updated website/cue/reference/administration/interfaces/kubectl.cue vector version to 0.55.0 * chore(releasing): Updated distribution/install.sh vector version to 0.55.0 * chore(releasing): Add 0.55.0 to versions.cue * chore(releasing): Created release md file * fix(api): restore HTTP GET /health endpoint (#25234) * refactor(api): serve gRPC via hyper + axum router Convert tonic's Server to an axum Router via into_router(), then serve over the same TcpListener via hyper::Server. Enables HTTP/1.1 acceptance so additional HTTP routes can be added alongside gRPC on the same port. Behavior-preserving. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(api): restore HTTP GET /health endpoint Re-expose the HTTP health endpoint that was removed as part of the GraphQL-to-gRPC migration (#24364). The endpoint matches the pre-0.55 response shape: 200 with body {"ok": true} while serving and 503 with body {"ok": false} after set_not_serving() is called during drain. HEAD is also handled. gRPC clients continue to use grpc.health.v1.Health/Check; both probes now share the same serving state so they agree during shutdown. Co-Authored-By: Claude Opus 4.7 (1M context) * test(api): cover HTTP GET/HEAD /health endpoint Adds two integration tests hitting the restored HTTP health endpoint via reqwest: - GET returns 200 with body {"ok":true} - HEAD returns 200 Exposes harness.api_port() so tests can reach the API port directly. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(api): document HTTP /health endpoint Document the HTTP GET/HEAD /health endpoint served alongside the gRPC API, framed as compatibility with Vector 0.54.0 and earlier. Updates the reference endpoints schema to allow HEAD, adds HEAD/GET entries for /health in api.cue with 200/503 responses, and adds a curl example to the API reference page. Co-Authored-By: Claude Opus 4.7 (1M context) * style(api): apply cargo fmt Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(demo_logs source): drop fakedata_generator, fix broken fake domains (#25236) * fix(demo_logs source): drop fakedata_generator, fix broken fake domains The fakedata_generator crate (v0.7.1) has an upstream bug: its get_dataset() match table is missing a "tlds" arm, so gen_domain() falls through to an empty JSON string. That fails to parse and the crate prints "Failed getting dataset for tlds. EOF while parsing a value at line 1 column 0" to stderr on every call, while gen_domain() returns ".Error: dataset not found" as the fake domain. Drop the dependency (and its transitive passt dep) and inline the three functions demo_logs actually used: gen_domain, gen_ipv4, gen_username. Domains now look like "random.io" instead of "random.Error: dataset not found", and stderr is quiet. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): changelog fragment for demo_logs fakedata fix Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): drop stale comment, allow TLDs/fakedata in spelling Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): swap fake domain name list to unique entries Replace the 8-word domain-name pool with fresh placeholder-style entries so nothing in lib/fakedata derives from the fakedata_generator crate's data. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(releasing): address Codex review feedback on 0.55 release cue - Strip trailing whitespace on blank lines in `0.55.0.cue` so `make check-fmt` passes. - Clarify that `azure_credential_kind` must live under the `azure_logs_ingestion` sink's `auth` block, not at the sink root, and link the 0.55 upgrade guide for a concrete example. Co-Authored-By: Claude Opus 4.7 (1M context) * update date --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(releasing): start 0.56.0 development cycle Bump version to 0.56.0 and switch VRL back to tracking the vectordotdev/vrl main branch after the 0.55.0 pin. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): regenerate Kubernetes manifests Output of cargo vdev build manifests for the 0.56.0 development cycle; picks up helm chart version 0.52.0. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): address cargo deny advisories for 0.56.0 - Upgrade rustls-webpki 0.103.12 -> 0.103.13 to pick up the fix for RUSTSEC-2026-0104 (reachable panic in CRL parsing). - Ignore RUSTSEC-2026-0104 for the still-reachable rustls-webpki 0.101 and 0.102, which are blocked on the tonic upgrade tracked in #19179. - Ignore RUSTSEC-2026-0105 (core2 unmaintained and yanked), reached transitively via libflate -> apache-avro; no safe upgrade available. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- Cargo.lock | 18 +- Cargo.toml | 2 +- .../23657-grpc-health-protocol.enhancement.md | 7 - .../24291_async_nats_websockets.feature.md | 4 - .../24411_geoip_network_field.enhancement.md | 3 - ..._source_per_signal_decoding.enhancement.md | 21 - .../24532_vector_stopped_log_ordering.fix.md | 5 - ..._dnstap_parser_message_size.enhancement.md | 3 - .../24593_graph_edge_attributes.feature.md | 3 - ...telemetry-header-enrichment.enhancement.md | 7 - ...9_kafka_sink_add_traces_support.feature.md | 3 - .../24641-task-transform-utilization.fix.md | 5 - changelog.d/24708_log_error.fix.md | 5 - ...24729_azure_blob_authentication.feature.md | 3 - .../24729_azure_logs_ingestion.breaking.md | 3 - ...-buffer-utilization-metric-tracking.fix.md | 3 - ...3_aggregate_transform_reduce_memory.fix.md | 4 - .../24951_top_events_out_column.fix.md | 3 - ...adog_metrics_zstd_series_v2.enhancement.md | 3 - ...fixed_high_cpu_usage_in_file_source.fix.md | 3 - .../aws_s3_parquet_encoding.feature.md | 10 - .../bump_kube_k8s_openapi.enhancement.md | 2 - ...ickhouse_arrow_uuid_support.enhancement.md | 3 - .../datadog_agent_source_device_tag_v2.fix.md | 5 - ...g_metrics_series_v2_default.enhancement.md | 5 - changelog.d/fix_site_prefix_stripping.fix.md | 3 - ...fix_source_lag_time_chunked_batches.fix.md | 3 - changelog.d/graphql_to_grpc_api.breaking.md | 42 -- ...deprecated_http_headers_option.breaking.md | 4 - ...urce_sender_latency_metrics.enhancement.md | 5 - ...or_top_disabled_memory_used.enhancement.md | 3 - .../windows_event_log_source.feature.md | 3 - .../windows_service_stop_state_wait.fix.md | 3 - deny.toml | 2 + distribution/install.sh | 2 +- .../kubernetes/vector-agent/README.md | 2 +- .../kubernetes/vector-agent/configmap.yaml | 5 +- .../kubernetes/vector-agent/daemonset.yaml | 8 +- .../kubernetes/vector-agent/rbac.yaml | 4 +- .../vector-agent/service-headless.yaml | 2 +- .../vector-agent/serviceaccount.yaml | 2 +- .../kubernetes/vector-aggregator/README.md | 2 +- .../vector-aggregator/configmap.yaml | 5 +- .../vector-aggregator/service-headless.yaml | 2 +- .../kubernetes/vector-aggregator/service.yaml | 2 +- .../vector-aggregator/serviceaccount.yaml | 2 +- .../vector-aggregator/statefulset.yaml | 8 +- .../vector-stateless-aggregator/README.md | 2 +- .../configmap.yaml | 5 +- .../deployment.yaml | 8 +- .../service-headless.yaml | 2 +- .../vector-stateless-aggregator/service.yaml | 2 +- .../serviceaccount.yaml | 2 +- .../2026-04-20-0-55-0-upgrade-guide.md | 139 +++++ website/content/en/releases/0.55.0.md | 4 + .../administration/interfaces/kubectl.cue | 2 +- website/cue/reference/releases/0.55.0.cue | 583 ++++++++++++++++++ website/cue/reference/versions.cue | 1 + website/layouts/releases/single.html | 22 +- 59 files changed, 786 insertions(+), 233 deletions(-) delete mode 100644 changelog.d/23657-grpc-health-protocol.enhancement.md delete mode 100644 changelog.d/24291_async_nats_websockets.feature.md delete mode 100644 changelog.d/24411_geoip_network_field.enhancement.md delete mode 100644 changelog.d/24455_otel_source_per_signal_decoding.enhancement.md delete mode 100644 changelog.d/24532_vector_stopped_log_ordering.fix.md delete mode 100644 changelog.d/24552_dnstap_parser_message_size.enhancement.md delete mode 100644 changelog.d/24593_graph_edge_attributes.feature.md delete mode 100644 changelog.d/24619-opentelemetry-header-enrichment.enhancement.md delete mode 100644 changelog.d/24639_kafka_sink_add_traces_support.feature.md delete mode 100644 changelog.d/24641-task-transform-utilization.fix.md delete mode 100644 changelog.d/24708_log_error.fix.md delete mode 100644 changelog.d/24729_azure_blob_authentication.feature.md delete mode 100644 changelog.d/24729_azure_logs_ingestion.breaking.md delete mode 100644 changelog.d/24911-buffer-utilization-metric-tracking.fix.md delete mode 100644 changelog.d/24943_aggregate_transform_reduce_memory.fix.md delete mode 100644 changelog.d/24951_top_events_out_column.fix.md delete mode 100644 changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md delete mode 100644 changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md delete mode 100644 changelog.d/aws_s3_parquet_encoding.feature.md delete mode 100644 changelog.d/bump_kube_k8s_openapi.enhancement.md delete mode 100644 changelog.d/clickhouse_arrow_uuid_support.enhancement.md delete mode 100644 changelog.d/datadog_agent_source_device_tag_v2.fix.md delete mode 100644 changelog.d/datadog_metrics_series_v2_default.enhancement.md delete mode 100644 changelog.d/fix_site_prefix_stripping.fix.md delete mode 100644 changelog.d/fix_source_lag_time_chunked_batches.fix.md delete mode 100644 changelog.d/graphql_to_grpc_api.breaking.md delete mode 100644 changelog.d/remove_deprecated_http_headers_option.breaking.md delete mode 100644 changelog.d/source_sender_latency_metrics.enhancement.md delete mode 100644 changelog.d/vector_top_disabled_memory_used.enhancement.md delete mode 100644 changelog.d/windows_event_log_source.feature.md delete mode 100644 changelog.d/windows_service_stop_state_wait.fix.md create mode 100644 website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md create mode 100644 website/content/en/releases/0.55.0.md create mode 100644 website/cue/reference/releases/0.55.0.cue diff --git a/Cargo.lock b/Cargo.lock index 2a7c05bd91410..a39a96a6500b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,7 +1781,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.106", "quote 1.0.44", "regex", @@ -8253,7 +8253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8299,7 +8299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -9473,7 +9473,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.12", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] @@ -9555,9 +9555,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -12275,7 +12275,7 @@ dependencies = [ [[package]] name = "vector" -version = "0.55.0" +version = "0.56.0" dependencies = [ "apache-avro 0.16.0", "approx", @@ -12912,8 +12912,8 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vrl" -version = "0.31.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#1357941d4481e84fb0323bd89c69cedf35c323f8" +version = "0.32.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#8a5b4711f37d0b3bd4d0d2ea8bbb500942bd8e98" dependencies = [ "aes", "aes-siv", diff --git a/Cargo.toml b/Cargo.toml index 03aff35ec8c13..1045e07aae1a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vector" -version = "0.55.0" +version = "0.56.0" authors = ["Vector Contributors "] edition = "2024" description = "A lightweight and ultra-fast tool for building observability pipelines" diff --git a/changelog.d/23657-grpc-health-protocol.enhancement.md b/changelog.d/23657-grpc-health-protocol.enhancement.md deleted file mode 100644 index a26e9ebf42b2c..0000000000000 --- a/changelog.d/23657-grpc-health-protocol.enhancement.md +++ /dev/null @@ -1,7 +0,0 @@ -`vector` source: Implement standard gRPC health checking protocol (`grpc.health.v1.Health`) -alongside the existing custom health check endpoint. This enables compatibility with standard -tools like `grpc-health-probe` for Kubernetes and other orchestration systems. - -Issue: https://github.com/vectordotdev/vector/issues/23657 - -authors: jpds diff --git a/changelog.d/24291_async_nats_websockets.feature.md b/changelog.d/24291_async_nats_websockets.feature.md deleted file mode 100644 index c11099ce6daf4..0000000000000 --- a/changelog.d/24291_async_nats_websockets.feature.md +++ /dev/null @@ -1,4 +0,0 @@ -Added websocket support to the `nats` source and sink. The `url` field now supports both `ws://` and -`wss://` protocols. - -authors: gedemagt diff --git a/changelog.d/24411_geoip_network_field.enhancement.md b/changelog.d/24411_geoip_network_field.enhancement.md deleted file mode 100644 index 65d33f50f6b1b..0000000000000 --- a/changelog.d/24411_geoip_network_field.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The `geoip` enrichment table now includes a `network` field containing the CIDR network associated with the lookup result, available for all database types (City, ISP/ASN, Connection-Type, Anonymous-IP). - -authors: naa0yama diff --git a/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md b/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md deleted file mode 100644 index 79db8934df463..0000000000000 --- a/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md +++ /dev/null @@ -1,21 +0,0 @@ -The `opentelemetry` source now supports independent configuration of OTLP decoding for logs, metrics, and traces. This allows more granular -control over which signal types are decoded, while maintaining backward compatibility with the existing boolean configuration. - -## Simple boolean form (applies to all signals) - -```yaml -use_otlp_decoding: true # All signals preserve OTLP format -# or -use_otlp_decoding: false # All signals use Vector native format (default) -``` - -## Per-signal configuration - -```yaml -use_otlp_decoding: - logs: false # Convert to Vector native format - metrics: false # Convert to Vector native format - traces: true # Preserve OTLP format -``` - -authors: pront diff --git a/changelog.d/24532_vector_stopped_log_ordering.fix.md b/changelog.d/24532_vector_stopped_log_ordering.fix.md deleted file mode 100644 index c5e395c37ab6f..0000000000000 --- a/changelog.d/24532_vector_stopped_log_ordering.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. - -A new `VectorStopping` event was added in the place of the `VectorStopped` event. - -authors: tronboto diff --git a/changelog.d/24552_dnstap_parser_message_size.enhancement.md b/changelog.d/24552_dnstap_parser_message_size.enhancement.md deleted file mode 100644 index 9472bde7f3346..0000000000000 --- a/changelog.d/24552_dnstap_parser_message_size.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Adds new fields to parsed dnstap data: `requestMessageSize` and `responseMessageSize`. It represents the size of the DNS message. - -authors: esensar Quad9DNS diff --git a/changelog.d/24593_graph_edge_attributes.feature.md b/changelog.d/24593_graph_edge_attributes.feature.md deleted file mode 100644 index 719cbc6b4635a..0000000000000 --- a/changelog.d/24593_graph_edge_attributes.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -`graph.edge_attributes` can now be added to transforms and sinks to add attributes to edges in graphs generated using `vector graph`. Memory enrichment tables are also considered for graphs, because they can have inputs and outputs. - -authors: esensar Quad9DNS diff --git a/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md b/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md deleted file mode 100644 index e0b0b33a5caa3..0000000000000 --- a/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md +++ /dev/null @@ -1,7 +0,0 @@ -`opentelemetry` source: Implemented header enrichment for OTLP metrics and traces. Unlike logs, which support enriching -the event itself or its metadata, depending on `log_namespace` settings, for metrics and traces this setting is ignored -and header values are added to the event metadata. - -Issue: https://github.com/vectordotdev/vector/issues/24619 - -authors: ozanichkovsky diff --git a/changelog.d/24639_kafka_sink_add_traces_support.feature.md b/changelog.d/24639_kafka_sink_add_traces_support.feature.md deleted file mode 100644 index b89dff7437ff3..0000000000000 --- a/changelog.d/24639_kafka_sink_add_traces_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The `kafka` sink now supports trace events. - -authors: pstalmach diff --git a/changelog.d/24641-task-transform-utilization.fix.md b/changelog.d/24641-task-transform-utilization.fix.md deleted file mode 100644 index f59a9c8a8eddf..0000000000000 --- a/changelog.d/24641-task-transform-utilization.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -Fixed utilization for task transforms to not account for time spent when downstream -is not polling. If the transform is frequently blocked on downstream components, -the reported utilization should be lower. - -authors: gwenaskell diff --git a/changelog.d/24708_log_error.fix.md b/changelog.d/24708_log_error.fix.md deleted file mode 100644 index d53626b09af05..0000000000000 --- a/changelog.d/24708_log_error.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -The `opentelemetry` source now logs an error if it fails to start up or during runtime. -This can happen when the configuration is invalid, for example trying to bind to the wrong -IP or when hitting the open file limit. - -authors: fbs diff --git a/changelog.d/24729_azure_blob_authentication.feature.md b/changelog.d/24729_azure_blob_authentication.feature.md deleted file mode 100644 index 7b224172563b7..0000000000000 --- a/changelog.d/24729_azure_blob_authentication.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. - -authors: jlaundry diff --git a/changelog.d/24729_azure_logs_ingestion.breaking.md b/changelog.d/24729_azure_logs_ingestion.breaking.md deleted file mode 100644 index ce0230a352f38..0000000000000 --- a/changelog.d/24729_azure_logs_ingestion.breaking.md +++ /dev/null @@ -1,3 +0,0 @@ -If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` to your sink config (this was previously the default, and now must be explicitly configured). - -authors: jlaundry diff --git a/changelog.d/24911-buffer-utilization-metric-tracking.fix.md b/changelog.d/24911-buffer-utilization-metric-tracking.fix.md deleted file mode 100644 index fd1c23e5414b6..0000000000000 --- a/changelog.d/24911-buffer-utilization-metric-tracking.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed regression in buffer utilization metric tracking around underflow. - -authors: bruceg diff --git a/changelog.d/24943_aggregate_transform_reduce_memory.fix.md b/changelog.d/24943_aggregate_transform_reduce_memory.fix.md deleted file mode 100644 index 4f3be38504bcd..0000000000000 --- a/changelog.d/24943_aggregate_transform_reduce_memory.fix.md +++ /dev/null @@ -1,4 +0,0 @@ -Reduced the memory usage in the `aggregate` transform where previous values were being held -even if `mode` was not set to `Diff`. - -authors: thomasqueirozb diff --git a/changelog.d/24951_top_events_out_column.fix.md b/changelog.d/24951_top_events_out_column.fix.md deleted file mode 100644 index e4191569cc205..0000000000000 --- a/changelog.d/24951_top_events_out_column.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed `vector top` displaying per-output sent events in the wrong column (Bytes In instead of Events Out) for components with multiple output ports. - -authors: pront diff --git a/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md b/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md deleted file mode 100644 index 1ecaef6b46b8c..0000000000000 --- a/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The `datadog_metrics` sink now uses zstd compression when submitting metrics to the Series v2 (`/api/v2/series`) and Sketches endpoints. Series v1 continues to use zlib (deflate). - -authors: vladimir-dd diff --git a/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md b/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md deleted file mode 100644 index 8d64593e51b5c..0000000000000 --- a/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an issue in the `file`/`kubernetes_logs` source that could cause unexpectedly high CPU usage after the async file server migration. - -authors: fcfangcc diff --git a/changelog.d/aws_s3_parquet_encoding.feature.md b/changelog.d/aws_s3_parquet_encoding.feature.md deleted file mode 100644 index afaa01cc70c26..0000000000000 --- a/changelog.d/aws_s3_parquet_encoding.feature.md +++ /dev/null @@ -1,10 +0,0 @@ -Add Apache Parquet batch encoding support for the `aws_s3` sink with flexible schema definitions. - -Events can now be encoded as Parquet columnar files with multiple schema input options: - -- **Native Parquet schema** — automatically generate a schema or supply `.schema` file -- **Configurable compression** - (Snappy, ZSTD, GZIP, LZ4, None). - -Enable the `codecs-parquet` feature and configure `batch_encoding` with `codec = "parquet"` in the S3 sink configuration. - -authors: szibis petere-datadog diff --git a/changelog.d/bump_kube_k8s_openapi.enhancement.md b/changelog.d/bump_kube_k8s_openapi.enhancement.md deleted file mode 100644 index 92b8fd24de684..0000000000000 --- a/changelog.d/bump_kube_k8s_openapi.enhancement.md +++ /dev/null @@ -1,2 +0,0 @@ -Bumped `kube` dependency from 0.93.0 to 3.0.1 and `k8s-openapi` from 0.22.0 to 0.27.0, adding support for Kubernetes API versions up to v1.35. -authors: hligit diff --git a/changelog.d/clickhouse_arrow_uuid_support.enhancement.md b/changelog.d/clickhouse_arrow_uuid_support.enhancement.md deleted file mode 100644 index e6dfc806b6775..0000000000000 --- a/changelog.d/clickhouse_arrow_uuid_support.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Added support for the ClickHouse `UUID` type in the ArrowStream format for the `clickhouse` sink. UUID columns are now automatically mapped to Arrow `Utf8` and cast by ClickHouse on insert. - -authors: benjamin-awd diff --git a/changelog.d/datadog_agent_source_device_tag_v2.fix.md b/changelog.d/datadog_agent_source_device_tag_v2.fix.md deleted file mode 100644 index 669d594f8c89f..0000000000000 --- a/changelog.d/datadog_agent_source_device_tag_v2.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -`datadog_agent` source: Preserve `device` as a plain tag when decoding v2 series metrics, -instead of incorrectly prefixing it as `resource.device`. This matches the v1 series behavior -and fixes tag remapping for disk, SNMP, and other integrations that use the `device` resource type. - -authors: lisaqvu diff --git a/changelog.d/datadog_metrics_series_v2_default.enhancement.md b/changelog.d/datadog_metrics_series_v2_default.enhancement.md deleted file mode 100644 index 39e857b73f99d..0000000000000 --- a/changelog.d/datadog_metrics_series_v2_default.enhancement.md +++ /dev/null @@ -1,5 +0,0 @@ -The `datadog_metrics` sink now defaults to the Datadog series v2 endpoint (`/api/v2/series`) and -exposes a new `series_api_version` configuration option (`v1` or `v2`) to control which endpoint is -used. Set `series_api_version: v1` to fall back to the legacy v1 endpoint if needed. - -authors: vladimir-dd diff --git a/changelog.d/fix_site_prefix_stripping.fix.md b/changelog.d/fix_site_prefix_stripping.fix.md deleted file mode 100644 index 8136664b5b2fd..0000000000000 --- a/changelog.d/fix_site_prefix_stripping.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed the Datadog sink healthcheck endpoint computation to preserve site prefixes (e.g. `us3.`, `us5.`, `ap1.`) when deriving the API URL from intake endpoints. Previously, the healthcheck for site-specific endpoints like `https://http-intake.logs.us3.datadoghq.com` would incorrectly call `https://api.datadoghq.com` instead of `https://api.us3.datadoghq.com`, causing unintended cross-site egress traffic. - -authors: vladimir-dd diff --git a/changelog.d/fix_source_lag_time_chunked_batches.fix.md b/changelog.d/fix_source_lag_time_chunked_batches.fix.md deleted file mode 100644 index 6303ba10edd33..0000000000000 --- a/changelog.d/fix_source_lag_time_chunked_batches.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an incorrect source_lag_time_seconds measurement in sources that use `send_batch` with large event batches. When a batch was split into multiple chunks, the reference timestamp used to compute lag time was re-captured on each chunk send, causing the lag time for later chunks to be overstated by the amount of time spent waiting for the channel to accept earlier chunks. The reference timestamp is now captured once before iteration and shared across all chunks. - -authors: gwenaskell diff --git a/changelog.d/graphql_to_grpc_api.breaking.md b/changelog.d/graphql_to_grpc_api.breaking.md deleted file mode 100644 index 2c9756ee16a89..0000000000000 --- a/changelog.d/graphql_to_grpc_api.breaking.md +++ /dev/null @@ -1,42 +0,0 @@ -The Vector observability API has been migrated from GraphQL to gRPC for improved -performance, efficiency and maintainability. The `vector top` and `vector tap` -commands continue to work as before, as they have been updated to use the new -gRPC API internally. The gRPC service definition is available in -[`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). - -Note: `vector top` and `vector tap` from version 0.55.0 or later are not -compatible with Vector instances running earlier versions. - -- Remove the `api.graphql` and `api.playground` fields from your config. Vector - now rejects configs that contain them. - -- If you use `vector top` or `vector tap` with an explicit `--url`, remove the - `/graphql` path suffix: - -```bash -# Old -vector top --url http://localhost:8686/graphql - -# New (the gRPC API listens at the root) -vector top --url http://localhost:8686 -``` - -- The GraphQL API (HTTP endpoint `/graphql`, WebSocket subscriptions, and the - GraphQL Playground at `/playground`) has been removed. You can interact with - the new gRPC API using tools like - [grpcurl](https://github.com/fullstorydev/grpcurl): - -```bash -# Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) -grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check - -# List components -grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents - -# Stream events (tap) — limit and interval_ms are required and must be >= 1 -grpcurl -plaintext \ - -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ - localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents -``` - -authors: pront diff --git a/changelog.d/remove_deprecated_http_headers_option.breaking.md b/changelog.d/remove_deprecated_http_headers_option.breaking.md deleted file mode 100644 index b2b1fd28b869e..0000000000000 --- a/changelog.d/remove_deprecated_http_headers_option.breaking.md +++ /dev/null @@ -1,4 +0,0 @@ -The `headers` option has been removed from the `http` and `opentelemetry` sinks. -Use `request.headers` instead. This option has been deprecated since v0.33.0. - -authors: thomasqueirozb diff --git a/changelog.d/source_sender_latency_metrics.enhancement.md b/changelog.d/source_sender_latency_metrics.enhancement.md deleted file mode 100644 index 6876641a37764..0000000000000 --- a/changelog.d/source_sender_latency_metrics.enhancement.md +++ /dev/null @@ -1,5 +0,0 @@ -Sources now record the distribution metrics `source_send_latency_seconds` (measuring the time spent -blocking on a single events chunk send operation on the output) and `source_send_batch_latency_seconds` -(encompassing all chunks within a received events batch). - -authors: gwenaskell diff --git a/changelog.d/vector_top_disabled_memory_used.enhancement.md b/changelog.d/vector_top_disabled_memory_used.enhancement.md deleted file mode 100644 index e929a065019ef..0000000000000 --- a/changelog.d/vector_top_disabled_memory_used.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -`vector top` terminal UI now shows `disabled` in the Memory Used column when the connected Vector instance was not started with `--allocation-tracing`, instead of displaying misleading zeros. A new `GetAllocationTracingStatus` gRPC endpoint is queried on connect to determine the status. - -authors: pront diff --git a/changelog.d/windows_event_log_source.feature.md b/changelog.d/windows_event_log_source.feature.md deleted file mode 100644 index 7f64f115a3391..0000000000000 --- a/changelog.d/windows_event_log_source.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added a new `windows_event_log` source that collects logs from Windows Event Log channels using the native Windows Event Log API with pull-mode subscriptions, bookmark-based checkpointing, and configurable field filtering. - -authors: tot19 diff --git a/changelog.d/windows_service_stop_state_wait.fix.md b/changelog.d/windows_service_stop_state_wait.fix.md deleted file mode 100644 index 45b60f01ca5fa..0000000000000 --- a/changelog.d/windows_service_stop_state_wait.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed Windows service state checks in `vector service start`/`stop`, and made `vector service stop` wait until the service reaches `Stopped`. Added `--stop-timeout` to `vector service stop` and `vector service uninstall`. - -authors: iMithrellas diff --git a/deny.toml b/deny.toml index ac8e588fa1cbe..0d563622a509e 100644 --- a/deny.toml +++ b/deny.toml @@ -48,4 +48,6 @@ ignore = [ { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 unsound with custom logger - transitive dependency, upstream crates have not updated to rand 0.9+" }, + { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, + { id = "RUSTSEC-2026-0105", reason = "core2 is unmaintained and all versions yanked - transitive dependency via libflate -> apache-avro, no safe upgrade available" }, ] diff --git a/distribution/install.sh b/distribution/install.sh index ceeb50c253862..f2ab269e2fd01 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.54.0"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.55.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index dfe0f05deb137..15dfbf6207643 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.52.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index e3d7499257236..99cb8d91f4345 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: kubernetes_logs: type: kubernetes_logs diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 4c468b88989a7..8c74a70107dec 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.55.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -58,6 +58,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 93b151826e238..5cd8e99594981 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 2f3a72b2619fd..573c2e67310c4 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 9db1dca796f94..c1b65be5aea95 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index 213cd06790f2e..ab45515cdaf94 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.52.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index 39fa1093bffa6..464539b0afd4d 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: datadog_agent: address: 0.0.0.0:8282 diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index 3170f1c3751cd..bd5a2c5cc8432 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index 3c423d4590371..5277bea614bdf 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index 4f97844c7fee1..8bc7b591147b7 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index d107f7d55cd91..e5ec9f1fcd1aa 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.55.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -67,6 +67,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index d03fb7eb5250b..d3a701364d611 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.52.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index e13b42240886c..5fe8fa6d1a4ca 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: datadog_agent: address: 0.0.0.0:8282 diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index 60e585952669b..d7b31fd50a07c 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.55.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -65,6 +65,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index bb2c35189c395..0c6143ee4074c 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index b96e58ef1ea31..2f325ba8df63a 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 2636f02bed059..0a282a591d5e7 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" automountServiceAccountToken: true diff --git a/website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md b/website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md new file mode 100644 index 0000000000000..ee06ab61b83fd --- /dev/null +++ b/website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md @@ -0,0 +1,139 @@ +--- +date: "2026-04-20" +title: "0.55 Upgrade Guide" +description: "An upgrade guide that addresses breaking changes in 0.55.0" +authors: ["pront"] +release: "0.55.0" +hide_on_release_notes: false +badges: + type: breaking change +--- + + +## Vector breaking changes + +1. [Observability API migrated from GraphQL to gRPC](#graphql-to-grpc) +2. [`headers` option removed from `http` and `opentelemetry` sinks](#http-headers-removed) +3. [`azure_logs_ingestion` sink requires explicit `azure_credential_kind`](#azure-logs-ingestion-credential) + +## Vector upgrade guide + +### Observability API migrated from GraphQL to gRPC {#graphql-to-grpc} + +Vector's observability API has been rewritten from GraphQL to gRPC. The following HTTP endpoints have been removed: + +- `/graphql` (HTTP POST queries and WebSocket subscriptions) +- `/playground` (the GraphQL Playground) + +The HTTP `GET /health` endpoint is unchanged and continues to serve Kubernetes HTTP liveness/readiness probes as before. + +Vector will now reject configs that contain `api.graphql` or `api.playground`. + +The gRPC service definition is available at [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). A standard [gRPC Health Checking Protocol](https://grpc.io/docs/guides/health-checking/) service (`grpc.health.v1.Health`) is also exposed for tooling that prefers gRPC probes, such as [`grpc-health-probe`](https://github.com/grpc-ecosystem/grpc-health-probe) or native Kubernetes gRPC probes. + +Note: `vector top` and `vector tap` from version 0.55.0 or later are not compatible with Vector instances running earlier versions. + +#### Action needed + +1. Please remove the `api.graphql` and `api.playground` fields from your Vector configuration. + +2. If you invoke `vector top` or `vector tap` with an explicit `--url`, please drop the `/graphql` suffix: + + ```bash + # Old + vector top --url http://localhost:8686/graphql + + # New (the gRPC API listens at the root) + vector top --url http://localhost:8686 + ``` + +3. If you interact with the API from external tooling, please migrate to the gRPC service. For example, using [`grpcurl`](https://github.com/fullstorydev/grpcurl): + + ```bash + # Check health (standard gRPC health check) + grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check + + # List components + grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents + + # Stream events (tap). Note: limit and interval_ms are required and must be >= 1 + grpcurl -plaintext \ + -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ + localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents + ``` + +### `headers` option removed from `http` and `opentelemetry` sinks {#http-headers-removed} + +The top-level `headers` option on the `http` and `opentelemetry` sinks has been removed. It was deprecated in v0.33.0 in favor of `request.headers`. + +#### Action needed + +Please move any `headers` values under `request.headers`. The exact location differs between the two sinks. + +For the `http` sink, `request` lives at the sink root: + +```yaml +# Old +sinks: + my_http: + type: http + uri: https://example.com + headers: + X-Custom-Header: value + +# New +sinks: + my_http: + type: http + uri: https://example.com + request: + headers: + X-Custom-Header: value +``` + +For the `opentelemetry` sink, `request` lives under the `protocol` block (because the sink wraps its transport under `protocol:`): + +```yaml +# Old +sinks: + my_otel: + type: opentelemetry + protocol: + type: http + uri: http://localhost:5318/v1/logs + headers: + X-Custom-Header: value + +# New +sinks: + my_otel: + type: opentelemetry + protocol: + type: http + uri: http://localhost:5318/v1/logs + request: + headers: + X-Custom-Header: value +``` + +Note: placing `request` at the `opentelemetry` sink root will not error, but the headers will be silently ignored. Please place `request` under `protocol`. + +### `azure_logs_ingestion` sink requires explicit `azure_credential_kind` {#azure-logs-ingestion-credential} + +The `azure_logs_ingestion` sink (introduced in 0.54.0) previously used `client_secret_credential` as the implicit default when Client Secret credentials were provided. That implicit default has been removed; `azure_credential_kind` must now be set explicitly. + +#### Action needed + +If you are using `azure_logs_ingestion` with Client Secret credentials, please add `azure_credential_kind: client_secret_credential` under the sink's `auth` block (alongside `azure_tenant_id`, `azure_client_id`, and `azure_client_secret`): + +```yaml +sinks: + my_azure_logs: + type: azure_logs_ingestion + # ... remaining config + auth: + azure_credential_kind: client_secret_credential + azure_tenant_id: "" + azure_client_id: "" + azure_client_secret: "" +``` diff --git a/website/content/en/releases/0.55.0.md b/website/content/en/releases/0.55.0.md new file mode 100644 index 0000000000000..55389520154a6 --- /dev/null +++ b/website/content/en/releases/0.55.0.md @@ -0,0 +1,4 @@ +--- +title: Vector v0.55.0 release notes +weight: 34 +--- diff --git a/website/cue/reference/administration/interfaces/kubectl.cue b/website/cue/reference/administration/interfaces/kubectl.cue index 32258dfb538b9..94d30ab6bedc2 100644 --- a/website/cue/reference/administration/interfaces/kubectl.cue +++ b/website/cue/reference/administration/interfaces/kubectl.cue @@ -19,7 +19,7 @@ administration: interfaces: kubectl: { role_implementations: [Name=string]: { commands: { _deployment_variant: string - _vector_version: "0.54" + _vector_version: "0.55" _namespace: string | *"vector" _controller_resource_type: string _controller_resource_name: string | *_deployment_variant diff --git a/website/cue/reference/releases/0.55.0.cue b/website/cue/reference/releases/0.55.0.cue new file mode 100644 index 0000000000000..1e022d2151a93 --- /dev/null +++ b/website/cue/reference/releases/0.55.0.cue @@ -0,0 +1,583 @@ +package metadata + +releases: "0.55.0": { + date: "2026-04-22" + codename: "" + + whats_next: [] + + description: """ + The [COSE team](https://opensource.datadoghq.com/about/#the-community-open-source-engineering-team) is excited to announce version `0.55.0`! + + ## Release highlights + + - New `windows_event_log` source that collects logs from Windows Event Log channels using the + native Windows Event Log API, with pull-mode subscriptions, bookmark-based checkpointing, and + configurable field filtering. + - The `aws_s3` sink now supports Apache Parquet batch encoding. Events can be written as + Parquet columnar files with either an auto-generated native schema or a supplied `.schema` + file, and configurable compression (Snappy, ZSTD, GZIP, LZ4, or none). + - The `azure_blob` sink re-gains first-class [Azure authentication](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-access-azure-active-directory): + Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion + credential kinds are all supported again. + - The `datadog_metrics` sink now defaults to the Series v2 endpoint (`/api/v2/series`) and + uses zstd compression for Series v2 and Sketches, which should yield smaller payloads + and more efficient batching and intake. A new `series_api_version` option (`v1` or `v2`) + is available to opt back to the legacy v1 endpoint; Series v1 continues to use zlib. + - `vector top` is more trustworthy: per-output events for components with multiple output + ports are now shown in the correct `Events Out` column, and the `Memory Used` column + now reports `disabled` when the target Vector instance was started without + `--allocation-tracing` instead of a misleading `0`. + - Better internal metrics for capacity planning and alerting: + - New source-send latency distributions (`source_send_latency_seconds`, + `source_send_batch_latency_seconds`) surface backpressure close to the source. + - Task-transform `utilization` no longer counts time spent waiting on downstream + components, giving a more representative view of transform saturation. + - Fixed a regression in buffer utilization metric tracking around underflow. + - Fixed a performance regression in the `file` and `kubernetes_logs` sources that could + cause unexpectedly high CPU usage, introduced in 0.50.0. + + ## Breaking Changes + + See the [0.55 upgrade guide](/highlights/2026-04-20-0-55-0-upgrade-guide/) for full details + and migration steps. At a glance, you are affected if you: + + - query or tail the Vector observability API in any way: the API has moved from + GraphQL to gRPC. This includes `vector top`, `vector tap`, and anything that talked to + `/graphql` or the `/playground`. The HTTP `GET /health` endpoint is unchanged and continues + to serve Kubernetes HTTP probes as before. + - set the top-level `headers` option on the `http` or `opentelemetry` sinks: it has been + removed. + - use the `azure_logs_ingestion` sink with Client Secret credentials: `azure_credential_kind` + must now be set explicitly. + """ + + changelog: [ + { + type: "enhancement" + description: #""" + `vector` source: Implement standard gRPC health checking protocol (`grpc.health.v1.Health`) + alongside the existing custom health check endpoint. This enables compatibility with standard + tools like `grpc-health-probe` for Kubernetes and other orchestration systems. + + Issue: https://github.com/vectordotdev/vector/issues/23657 + """# + contributors: ["jpds"] + }, + { + type: "feat" + description: #""" + Added websocket support to the `nats` source and sink. The `url` field now supports both `ws://` and + `wss://` protocols. + """# + contributors: ["gedemagt"] + }, + { + type: "enhancement" + description: #""" + The `geoip` enrichment table now includes a `network` field containing the CIDR network associated with the lookup result, available for all database types (City, ISP/ASN, Connection-Type, Anonymous-IP). + """# + contributors: ["naa0yama"] + }, + { + type: "enhancement" + description: #""" + The `opentelemetry` source now supports independent configuration of OTLP decoding for logs, metrics, and traces. This allows more granular + control over which signal types are decoded, while maintaining backward compatibility with the existing boolean configuration. + + ## Simple boolean form (applies to all signals) + + ```yaml + use_otlp_decoding: true # All signals preserve OTLP format + # or + use_otlp_decoding: false # All signals use Vector native format (default) + ``` + + ## Per-signal configuration + + ```yaml + use_otlp_decoding: + logs: false # Convert to Vector native format + metrics: false # Convert to Vector native format + traces: true # Preserve OTLP format + ``` + """# + contributors: ["pront"] + }, + { + type: "fix" + description: #""" + Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. + + A new `VectorStopping` event was added in the place of the `VectorStopped` event. + """# + contributors: ["tronboto"] + }, + { + type: "enhancement" + description: #""" + Adds new fields to parsed dnstap data: `requestMessageSize` and `responseMessageSize`. It represents the size of the DNS message. + """# + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "feat" + description: #""" + `graph.edge_attributes` can now be added to transforms and sinks to add attributes to edges in graphs generated using `vector graph`. Memory enrichment tables are also considered for graphs, because they can have inputs and outputs. + """# + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "enhancement" + description: #""" + `opentelemetry` source: Implemented header enrichment for OTLP metrics and traces. Unlike logs, which support enriching + the event itself or its metadata, depending on `log_namespace` settings, for metrics and traces this setting is ignored + and header values are added to the event metadata. + + Issue: https://github.com/vectordotdev/vector/issues/24619 + """# + contributors: ["ozanichkovsky"] + }, + { + type: "feat" + description: #""" + The `kafka` sink now supports trace events. + """# + contributors: ["pstalmach"] + }, + { + type: "fix" + description: #""" + Fixed utilization for task transforms to not account for time spent when downstream + is not polling. If the transform is frequently blocked on downstream components, + the reported utilization should be lower. + """# + contributors: ["gwenaskell"] + }, + { + type: "fix" + description: #""" + The `opentelemetry` source now logs an error if it fails to start up or during runtime. + This can happen when the configuration is invalid, for example trying to bind to the wrong + IP or when hitting the open file limit. + """# + contributors: ["fbs"] + }, + { + type: "feat" + description: #""" + Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. + """# + contributors: ["jlaundry"] + }, + { + type: "chore" + description: #""" + If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` under the sink's `auth` block (alongside `azure_tenant_id`, `azure_client_id`, and `azure_client_secret`). This was previously the default, and now must be explicitly configured. See the [0.55 upgrade guide](/highlights/2026-04-20-0-55-0-upgrade-guide/) for an example. + """# + contributors: ["jlaundry"] + }, + { + type: "fix" + description: #""" + Fixed regression in buffer utilization metric tracking around underflow. + """# + contributors: ["bruceg"] + }, + { + type: "fix" + description: #""" + Reduced the memory usage in the `aggregate` transform where previous values were being held + even if `mode` was not set to `Diff`. + """# + contributors: ["thomasqueirozb"] + }, + { + type: "fix" + description: #""" + Fixed `vector top` displaying per-output sent events in the wrong column (Bytes In instead of Events Out) for components with multiple output ports. + """# + contributors: ["pront"] + }, + { + type: "enhancement" + description: #""" + The `datadog_metrics` sink now uses zstd compression when submitting metrics to the Series v2 (`/api/v2/series`) and Sketches endpoints. Series v1 continues to use zlib (deflate). + """# + contributors: ["vladimir-dd"] + }, + { + type: "fix" + description: #""" + Fixed an issue in the `file`/`kubernetes_logs` source that could cause unexpectedly high CPU usage after the async file server migration. + """# + contributors: ["fcfangcc"] + }, + { + type: "feat" + description: #""" + Add Apache Parquet batch encoding support for the `aws_s3` sink with flexible schema definitions. + + Events can now be encoded as Parquet columnar files with multiple schema input options: + + - **Native Parquet schema** — automatically generate a schema or supply `.schema` file + - **Configurable compression** - (Snappy, ZSTD, GZIP, LZ4, None). + + Enable the `codecs-parquet` feature and configure `batch_encoding` with `codec = "parquet"` in the S3 sink configuration. + """# + contributors: ["szibis", "petere-datadog"] + }, + { + type: "enhancement" + description: #""" + Bumped `kube` dependency from 0.93.0 to 3.0.1 and `k8s-openapi` from 0.22.0 to 0.27.0, adding support for Kubernetes API versions up to v1.35. + """# + contributors: ["hligit"] + }, + { + type: "enhancement" + description: #""" + Added support for the ClickHouse `UUID` type in the ArrowStream format for the `clickhouse` sink. UUID columns are now automatically mapped to Arrow `Utf8` and cast by ClickHouse on insert. + """# + contributors: ["benjamin-awd"] + }, + { + type: "fix" + description: #""" + `datadog_agent` source: Preserve `device` as a plain tag when decoding v2 series metrics, + instead of incorrectly prefixing it as `resource.device`. This matches the v1 series behavior + and fixes tag remapping for disk, SNMP, and other integrations that use the `device` resource type. + """# + contributors: ["lisaqvu"] + }, + { + type: "enhancement" + description: #""" + The `datadog_metrics` sink now defaults to the Datadog series v2 endpoint (`/api/v2/series`) and + exposes a new `series_api_version` configuration option (`v1` or `v2`) to control which endpoint is + used. Set `series_api_version: v1` to fall back to the legacy v1 endpoint if needed. + """# + contributors: ["vladimir-dd"] + }, + { + type: "fix" + description: #""" + Fixed the Datadog sink healthcheck endpoint computation to preserve site prefixes (e.g. `us3.`, `us5.`, `ap1.`) when deriving the API URL from intake endpoints. Previously, the healthcheck for site-specific endpoints like `https://http-intake.logs.us3.datadoghq.com` would incorrectly call `https://api.datadoghq.com` instead of `https://api.us3.datadoghq.com`, causing unintended cross-site egress traffic. + """# + contributors: ["vladimir-dd"] + }, + { + type: "fix" + description: #""" + Fixed an incorrect source_lag_time_seconds measurement in sources that use `send_batch` with large event batches. When a batch was split into multiple chunks, the reference timestamp used to compute lag time was re-captured on each chunk send, causing the lag time for later chunks to be overstated by the amount of time spent waiting for the channel to accept earlier chunks. The reference timestamp is now captured once before iteration and shared across all chunks. + """# + contributors: ["gwenaskell"] + }, + { + type: "chore" + description: #""" + The Vector observability API has been migrated from GraphQL to gRPC for improved + performance, efficiency and maintainability. The `vector top` and `vector tap` + commands continue to work as before, as they have been updated to use the new + gRPC API internally. The gRPC service definition is available in + [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). + + Note: `vector top` and `vector tap` from version 0.55.0 or later are not + compatible with Vector instances running earlier versions. + + - Remove the `api.graphql` and `api.playground` fields from your config. Vector + now rejects configs that contain them. + + - If you use `vector top` or `vector tap` with an explicit `--url`, remove the + `/graphql` path suffix: + + ```bash + # Old + vector top --url http://localhost:8686/graphql + + # New (the gRPC API listens at the root) + vector top --url http://localhost:8686 + ``` + + - The GraphQL API (HTTP endpoint `/graphql`, WebSocket subscriptions, and the + GraphQL Playground at `/playground`) has been removed. You can interact with + the new gRPC API using tools like + [grpcurl](https://github.com/fullstorydev/grpcurl): + + ```bash + # Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) + grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check + + # List components + grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents + + # Stream events (tap) — limit and interval_ms are required and must be >= 1 + grpcurl -plaintext \ + -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ + localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents + ``` + """# + contributors: ["pront"] + }, + { + type: "chore" + description: #""" + The `headers` option has been removed from the `http` and `opentelemetry` sinks. + Use `request.headers` instead. On the `opentelemetry` sink, `request` is nested under + `protocol`; see the [0.55 upgrade guide](/highlights/2026-04-20-0-55-0-upgrade-guide/) + for examples. This option has been deprecated since v0.33.0. + """# + contributors: ["thomasqueirozb"] + }, + { + type: "enhancement" + description: #""" + Sources now record the distribution metrics `source_send_latency_seconds` (measuring the time spent + blocking on a single events chunk send operation on the output) and `source_send_batch_latency_seconds` + (encompassing all chunks within a received events batch). + """# + contributors: ["gwenaskell"] + }, + { + type: "enhancement" + description: #""" + `vector top` terminal UI now shows `disabled` in the Memory Used column when the connected Vector instance was not started with `--allocation-tracing`, instead of displaying misleading zeros. A new `GetAllocationTracingStatus` gRPC endpoint is queried on connect to determine the status. + """# + contributors: ["pront"] + }, + { + type: "feat" + description: #""" + Added a new `windows_event_log` source that collects logs from Windows Event Log channels using the native Windows Event Log API with pull-mode subscriptions, bookmark-based checkpointing, and configurable field filtering. + """# + contributors: ["tot19"] + }, + { + type: "fix" + description: #""" + Fixed Windows service state checks in `vector service start`/`stop`, and made `vector service stop` wait until the service reaches `Stopped`. Added `--stop-timeout` to `vector service stop` and `vector service uninstall`. + """# + contributors: ["iMithrellas"] + }, + { + type: "fix" + description: #""" + The text content generated by the `demo_logs` source has changed: the + pool of fake usernames and the pool of fake domain TLDs are now both + defined inside Vector rather than pulled from an external crate. The + line formats (`apache_common`, `apache_error`, `json`, `syslog`, + `bsd_syslog`) are unchanged. If any of your tests or downstream + pipelines assert on specific generated usernames or TLDs, please + update those expectations. + """# + contributors: ["pront"] + }, + ] + + vrl_changelog: """ + ### [0.32.0 (2026-04-16)] + + #### New Features + + - Added a new `encode_csv` function that encodes an array of values into a CSV-formatted string. This is the inverse of the existing `parse_csv` function and supports an optional single-byte delimiter (defaults to `,`). + + authors: armleth (https://github.com/vectordotdev/vrl/pull/1649) + - Added `to_entries` and `from_entries` with jq-compatible behavior: `to_entries` supports both objects and arrays, and `from_entries` accepts `key`/`Key`/`name`/`Name` and `value`/`Value` aliases. + + authors: close2code-palm (https://github.com/vectordotdev/vrl/pull/1653) + + #### Enhancements + + - Added `except` parameter to `flatten` function to exclude specific keys from being flattened. + + authors: benjamin-awd (https://github.com/vectordotdev/vrl/pull/1682) + + #### Fixes + + - Fixed a bug where the REPL input validator was executing programs instead of only compiling them, causing functions with side effects (e.g. `http_request`) to run twice per submission. + + authors: prontidis (https://github.com/vectordotdev/vrl/pull/1701) + + + ### [0.31.0 (2026-03-05)] + """ + + commits: [ + {sha: "bc30368b01670480b4e83bebdd9eb09ec78bd2e3", date: "2026-03-10 01:08:29 UTC", description: "make build vrl-docs work with released VRL version", pr_number: 24877, scopes: ["vdev"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 21, deletions_count: 19}, + {sha: "3f98053f5105d0bedbd9b1cb0f15917b22819e36", date: "2026-03-10 01:42:09 UTC", description: "use cross-strip tools when packaging RPMs for non-x86_64 targets", pr_number: 24873, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 29, deletions_count: 4}, + {sha: "0d462e06d4fd1088cd7054cd9ee7c49f91fe9b65", date: "2026-03-10 19:46:12 UTC", description: "increase timeouts for vdev compilation", pr_number: 24881, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 4, deletions_count: 4}, + {sha: "e66270f937998848a439252d46a160ad085b17c6", date: "2026-03-10 19:52:15 UTC", description: "add retries to address choco intermittent failures", pr_number: 24880, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 26, deletions_count: 5}, + {sha: "145d333fc0fc8a53a087c64c56775ff55612d4a8", date: "2026-03-10 20:36:38 UTC", description: "remove unused compilation timings workflow", pr_number: 24882, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 0, deletions_count: 77}, + {sha: "685c31fff163575019ce315b4b88991b92e3a61f", date: "2026-03-10 18:09:30 UTC", description: "pin actions to sha", pr_number: 24884, scopes: ["ci"], type: "chore", breaking_change: false, author: "StepSecurity Bot", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "40143b32740e253be37b214cf9f860851f65e6e0", date: "2026-03-10 18:31:15 UTC", description: "pin image tags in Dockerfiles", pr_number: 24885, scopes: ["ci"], type: "chore", breaking_change: false, author: "StepSecurity Bot", files_count: 7, insertions_count: 11, deletions_count: 11}, + {sha: "b1c7d7bf7133c43e472d15b4e4345e571043d63a", date: "2026-03-10 22:59:33 UTC", description: "fix broken links", pr_number: 24886, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 15, insertions_count: 18, deletions_count: 18}, + {sha: "1cc1c25147bab5ead963ff8065086d3b5daa1b2f", date: "2026-03-11 00:13:05 UTC", description: "minor fixes to release template", pr_number: 24887, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 13, deletions_count: 13}, + {sha: "e1ecad3d67361dd38f074e71da45d4d90b3578bb", date: "2026-03-11 00:37:56 UTC", description: "add docs::warnings macro support and warn about no auth", pr_number: 24866, scopes: ["website"], type: "feat", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 15, deletions_count: 1}, + {sha: "909b08318dd309c8a59e1c40241be42f46e67db8", date: "2026-03-11 00:53:15 UTC", description: "0.54.0 post release steps", pr_number: 24888, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 51, insertions_count: 512, deletions_count: 144}, + {sha: "df396d18b6101f1af5a75bbc918accb9291a7a74", date: "2026-03-11 18:01:29 UTC", description: "minor fixes to release changelog", pr_number: 24893, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 4}, + {sha: "356409dba6cc78cdaa0d070941efd9c0d52b1579", date: "2026-03-11 23:53:27 UTC", description: "test_udp_syslog can overflow default size receive buffer", pr_number: 24878, scopes: ["unit tests"], type: "fix", breaking_change: false, author: "strophy", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "901a571712d7714f5ad38defb60b01dbabe43617", date: "2026-03-11 19:11:14 UTC", description: "show top level api configuration in API reference page", pr_number: 24894, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 1, deletions_count: 3}, + {sha: "202b65612ee4cc27a338fc16a94d880c01b8ace6", date: "2026-03-11 19:34:07 UTC", description: "fix docker warning when publishing", pr_number: 24895, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 0, deletions_count: 1}, + {sha: "2958f982b221060b68dcb63837823837d4c0600a", date: "2026-03-11 22:57:01 UTC", description: "fetch full history in changelog workflow so origin/master is available", pr_number: 24898, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 0}, + {sha: "d112a1b6826d0a35ff27be40d949d09ee66b6a7e", date: "2026-03-11 23:17:02 UTC", description: "Support per-signal OTLP decoding config", pr_number: 24822, scopes: ["opentelemetry source"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 472, deletions_count: 24}, + {sha: "ff7164a43992728f3490e38dbf72e5125b162f50", date: "2026-03-12 00:55:04 UTC", description: "bump protoc version", pr_number: 24902, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "71d597f894f59ff6190913b1091a1c568fedba63", date: "2026-03-12 17:15:23 UTC", description: "remove unused Hugo shortcodes", pr_number: 24903, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 8, insertions_count: 0, deletions_count: 296}, + {sha: "40f94ee88a453a6a900a00606fc940d990c5143e", date: "2026-03-12 19:20:22 UTC", description: "bump VRL and resolve RUSTSEC-2021-0139", pr_number: 24908, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 11, insertions_count: 59, deletions_count: 94}, + {sha: "478d7c4561a1a8b2390e97973ec9315963dd0dfb", date: "2026-03-13 10:54:17 UTC", description: "bump kube from 0.93.0 to 3.0.1 and k8s-openapi from 0.22.0 to 0.27.0", pr_number: 24787, scopes: ["deps"], type: "chore", breaking_change: false, author: "Haitao Li", files_count: 8, insertions_count: 146, deletions_count: 140}, + {sha: "958ff6357a6ad700d592aabf53c6b161f5fbb9a0", date: "2026-03-12 21:54:25 UTC", description: "replace netlink-* crates with procfs", pr_number: 24765, scopes: ["deps"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 148, deletions_count: 369}, + {sha: "d141e60bfcc923a6d2df34c16f49db5fc0734e3c", date: "2026-03-12 23:40:12 UTC", description: "deprecate azure_monitor_logs sink", pr_number: 24910, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 0}, + {sha: "01c86e826a83175002b609b9a404325931f11cc1", date: "2026-03-13 21:24:47 UTC", description: "bump async-executor to 1.14.0", pr_number: 24919, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 8, deletions_count: 5}, + {sha: "891643edcb177b9cc1e8db8fb753b36ad07f9182", date: "2026-03-14 10:54:10 UTC", description: "Add Windows Event Log source", pr_number: 24305, scopes: ["new source"], type: "feat", breaking_change: false, author: "tot19", files_count: 26, insertions_count: 10298, deletions_count: 14}, + {sha: "4e958028e836a62080d70ac1099d7db635d5b739", date: "2026-03-14 02:38:46 UTC", description: "implement standard gRPC health checking protocol", pr_number: 24916, scopes: ["vector source"], type: "fix", breaking_change: false, author: "Jonathan Davies", files_count: 6, insertions_count: 139, deletions_count: 7}, + {sha: "2064a15aa1029aec49dd2b4c8b3a09a4b76ff9e7", date: "2026-03-13 22:55:11 UTC", description: "update dd-rust-license-tool from 1.0.5 to 1.0.6", pr_number: 24920, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 6, deletions_count: 1}, + {sha: "60cd0e1347d4ae680da806401a6c89be1a2c0536", date: "2026-03-14 00:09:55 UTC", description: "Bump rmp from 0.8.14 to 0.8.15 and resolve RUSTSEC-2024-0436", pr_number: 24922, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 3, deletions_count: 13}, + {sha: "da85874bd56ee88699778fb3ed3f4f847915581c", date: "2026-03-14 00:46:55 UTC", description: "bump aws crates and resolve RUSTSEC-2026-0002", pr_number: 24909, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 5, insertions_count: 277, deletions_count: 178}, + {sha: "59f53e2bc72caae239970b355b4edab14176cd17", date: "2026-03-14 01:11:33 UTC", description: "fix typos in rustdoc comments", pr_number: 24923, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 4, deletions_count: 4}, + {sha: "89bc15fc1815572000d1fd2e9e20c4b98116d1b2", date: "2026-03-14 05:17:33 UTC", description: "Bump lapin from 2.5.3 to 4.3.0", pr_number: 23316, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 8, insertions_count: 183, deletions_count: 156}, + {sha: "51aebc8445fece5e0e7daf62603bf004b9dfb8ce", date: "2026-03-16 22:18:15 UTC", description: "bump undici from 7.18.2 to 7.24.1 in /website", pr_number: 24925, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "e3225205fa88d04c4d76d0e485f27dc771900693", date: "2026-03-17 06:31:01 UTC", description: "add UUID type support for ArrowStream", pr_number: 24856, scopes: ["clickhouse sink"], type: "enhancement", breaking_change: false, author: "Benjamin Dornel", files_count: 4, insertions_count: 43, deletions_count: 4}, + {sha: "7390fe0bd4922b6f804b7c59e56dac36747af86d", date: "2026-03-16 19:08:19 UTC", description: "document Cargo feature placement rules", pr_number: 24933, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 25, deletions_count: 0}, + {sha: "c27f6d6dd8acc01f57e93c53af4fad0138a07aa1", date: "2026-03-16 20:05:44 UTC", description: "update heim fork to remove unmaintained crates", pr_number: 24924, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 4, insertions_count: 71, deletions_count: 214}, + {sha: "0d8bb9bea3418bd6993e08b456fc495050a42930", date: "2026-03-17 01:07:42 UTC", description: "bump pulsar from 6.3.1 to 6.7.0", pr_number: 24746, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 24, deletions_count: 23}, + {sha: "fd766a0738802e9dee2eae1e5d6f243ed9c69229", date: "2026-03-16 21:10:58 UTC", description: "make rdkafka use gssapi-vendored when publishing/testing", pr_number: 24912, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 4, insertions_count: 18, deletions_count: 10}, + {sha: "d38425ad1302489d7420a9d24447224559a5a446", date: "2026-03-17 01:16:13 UTC", description: "bump regex from 1.11.2 to 1.12.3", pr_number: 24796, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 13, deletions_count: 13}, + {sha: "c2fc606306e10e02263d2982f20c8c17f69a143b", date: "2026-03-16 22:05:29 UTC", description: "check Cargo.lock for changes after cargo check/clippy", pr_number: 24935, scopes: ["vdev"], type: "feat", breaking_change: false, author: "Thomas", files_count: 6, insertions_count: 57, deletions_count: 12}, + {sha: "0573297e8ffcc1ad7a096f50f012b2efd0518c27", date: "2026-03-17 03:54:36 UTC", description: "Added feature flag for websockets in async-nats", pr_number: 24291, scopes: ["nats source", "nats sink"], type: "feat", breaking_change: false, author: "Jesper", files_count: 3, insertions_count: 6, deletions_count: 1}, + {sha: "3905b4f12067fb2508f0bcb031607da84623f17f", date: "2026-03-16 23:07:53 UTC", description: "Update README.md and cleanup unused commands", pr_number: 24936, scopes: ["vdev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 7, insertions_count: 43, deletions_count: 95}, + {sha: "1eca246d5bf297afcc153b555c19ac5f9e9ff372", date: "2026-03-17 00:49:29 UTC", description: "replace derivative macro with standard Default derives on enums", pr_number: 24938, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 31, insertions_count: 74, deletions_count: 119}, + {sha: "a8730894c366cd5232353530fef4a4970cd6c307", date: "2026-03-17 19:00:09 UTC", description: "bump lz4_flex from 0.11.5 to 0.11.6", pr_number: 24939, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "9dc3d3e9f7d486ce363975f21b1e1b3b537cb46d", date: "2026-03-17 19:11:36 UTC", description: "reduce memory when not in `Diff` mode", pr_number: 24943, scopes: ["aggregate transform"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 89, deletions_count: 24}, + {sha: "42b844e76882fa5cbf1fd5cfafcf653101dc9ad0", date: "2026-03-17 19:13:06 UTC", description: "remove unused fs package", pr_number: 24945, scopes: ["website deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 0, deletions_count: 6}, + {sha: "218aa4d29b7b0922f604b7f6d8a3ebd0cc8e0931", date: "2026-03-17 20:01:18 UTC", description: "Add osv-scanner.toml and ignore MAL-2025-3174", pr_number: 24946, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "83bcf4cb297f85525f88a3cff1870ab8fbddd47b", date: "2026-03-17 20:23:49 UTC", description: "bump quinn-proto from 0.11.9 to 0.11.14", pr_number: 24926, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 12, deletions_count: 4}, + {sha: "dc9d8de0c8f2ff7a04a7373820f1ef652ed30520", date: "2026-03-18 03:29:20 UTC", description: "switch to v2 endpoint", pr_number: 24842, scopes: ["datadog_metrics sink"], type: "chore", breaking_change: false, author: "Vladimir Zhuk", files_count: 32, insertions_count: 382, deletions_count: 98}, + {sha: "1444b2bac009aa51343820dba7a36ce119dce56f", date: "2026-03-17 22:44:09 UTC", description: "Bump rdkafka from 0.38.0 to 0.39.0", pr_number: 24602, scopes: ["deps"], type: "chore", breaking_change: false, author: "zapdos26", files_count: 4, insertions_count: 22, deletions_count: 5}, + {sha: "089db6775500eb6cbdfa5f4b7e6069185a8d61c5", date: "2026-03-18 11:45:28 UTC", description: "add `network` CIDR field to lookup results", pr_number: 24576, scopes: ["enrichment tables"], type: "feat", breaking_change: false, author: "Naoki Aoyama", files_count: 5, insertions_count: 40, deletions_count: 8}, + {sha: "5011c5dfd20a0200c37a42a6c3001ae3df94ef4d", date: "2026-03-17 23:16:34 UTC", description: "bump docker/build-push-action from 6.18.0 to 6.19.2", pr_number: 24734, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 4, deletions_count: 4}, + {sha: "6527346294203cec7fb0d8b86c7fc98b26c14ff0", date: "2026-03-18 03:17:07 UTC", description: "bump aws-actions/configure-aws-credentials from 5.1.1 to 6.0.0", pr_number: 24733, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 6, deletions_count: 6}, + {sha: "b1a43a4ba24b0e76e57736953a0f6f9d0ee58328", date: "2026-03-18 04:52:26 UTC", description: "bump sysinfo from 0.37.2 to 0.38.2", pr_number: 24816, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 61, deletions_count: 39}, + {sha: "cfee4fb9d1853a5ae7db85e0bd5647ca7a7b57b0", date: "2026-03-18 00:54:19 UTC", description: "upgrade mold linker from 1.2.1/2.0.0 to 2.40.4", pr_number: 24947, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 3, deletions_count: 3}, + {sha: "fbb1e4b501d6081d72f864da891a98464e60d097", date: "2026-03-18 02:05:08 UTC", description: "Improve buffer utilization metric tracking", pr_number: 24911, scopes: ["buffers"], type: "fix", breaking_change: false, author: "Bruce Guenter", files_count: 2, insertions_count: 66, deletions_count: 95}, + {sha: "95f439420ae1dea0df7fbd295e7f354b144360c8", date: "2026-03-18 18:25:48 UTC", description: "place per-output sent events in Events Out column", pr_number: 24951, scopes: ["api"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 8, deletions_count: 3}, + {sha: "30936678f92a03c827f9fd8506fa02693332f59b", date: "2026-03-19 03:34:43 UTC", description: "add support for edge_attributes in graph configuration", pr_number: 24593, scopes: ["cli"], type: "feat", breaking_change: false, author: "Ensar Sarajčić", files_count: 7, insertions_count: 505, deletions_count: 125}, + {sha: "a8f3a9b45e6d2f6a897995bce85e3ad9a9a13442", date: "2026-03-19 03:39:25 UTC", description: "expose message size when parsing dnstap data", pr_number: 24552, scopes: ["dnstap source"], type: "enhancement", breaking_change: false, author: "Ensar Sarajčić", files_count: 4, insertions_count: 51, deletions_count: 0}, + {sha: "f34e8d32140ab25024ff51863ed9b5038defdc80", date: "2026-03-19 17:13:57 UTC", description: "regenerate VRL docs with message size fields", pr_number: 24960, scopes: ["dnstap source"], type: "docs", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 2, deletions_count: 0}, + {sha: "483db702e3745f4b823edac33e1d76bf7060f964", date: "2026-03-19 22:16:35 UTC", description: "add libsasl2-2 runtime dep to Debian-based images", pr_number: 24962, scopes: ["releasing"], type: "fix", breaking_change: false, author: "Vladimir Zhuk", files_count: 2, insertions_count: 2, deletions_count: 2}, + {sha: "5f0f03e181a031ee51a71e2bd9fcf0698f073a5b", date: "2026-03-19 23:13:14 UTC", description: "bump fakedata_generator from 0.5.0 to 0.7.1", pr_number: 24800, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 5, deletions_count: 5}, + {sha: "3c69a8b8a668c6d20ea57d6883a3771803893648", date: "2026-03-19 19:43:24 UTC", description: "instruct agents to use PR template when creating PRs", pr_number: 24965, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 4, deletions_count: 0}, + {sha: "f02424389cfa4abf8e1168b5b63fb227499c9bd4", date: "2026-03-19 19:54:48 UTC", description: "add Datadog static analysis workflow", pr_number: 24966, scopes: ["ci"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 26, deletions_count: 0}, + {sha: "a7fae1f7ad17b51e3960850300b45955ac0952e0", date: "2026-03-19 20:27:12 UTC", description: "add code coverage workflow with Datadog upload", pr_number: 24964, scopes: ["ci"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 54, deletions_count: 2}, + {sha: "37a48ae53eb0e5930f4f4506903eb5b10c613789", date: "2026-03-20 05:44:04 UTC", description: "bump cargo-lock from 10.1.0 to 11.0.1", pr_number: 24748, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 11, deletions_count: 37}, + {sha: "97dfb6c6a870a162d07827706e85f5ee2fe3b3a9", date: "2026-03-20 17:53:12 UTC", description: "bump const-str from 1.0.0 to 1.1.0", pr_number: 24815, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "18898e0149f43924a07554303cce3d0a4595eb5b", date: "2026-03-20 21:22:10 UTC", description: "bump quickcheck to 1.1.0 and use workspace dep", pr_number: 24970, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 8, insertions_count: 218, deletions_count: 31}, + {sha: "2097fc35d7dc8d3caee9607d563165374e312e21", date: "2026-03-21 03:12:17 UTC", description: "bump criterion from 0.7.0 to 0.8.2", pr_number: 24803, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 6, insertions_count: 31, deletions_count: 9}, + {sha: "2c334d223578ea17d7ca5bbbf305f21c9b68a7e2", date: "2026-03-21 05:07:16 UTC", description: "Add support for traces in kafka sink", pr_number: 24639, scopes: ["kafka sink"], type: "enhancement", breaking_change: false, author: "Piotr Stalmach", files_count: 4, insertions_count: 160, deletions_count: 12}, + {sha: "c748d6f9e19632cb64f666058794aa2d9004926f", date: "2026-03-21 02:28:58 UTC", description: "use cargo nextest in coverage workflow to prevent test pollution", pr_number: 24973, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 4, deletions_count: 1}, + {sha: "4d8ca64dbbbc2434bd4a69a3d46705c3fad6a56c", date: "2026-03-23 21:53:30 UTC", description: "pin localstack image to SHA256 digest", pr_number: 24988, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "6ef3752d12be1e8de38449df30f953f75301f64f", date: "2026-03-23 20:52:03 UTC", description: "remove futures features and resolve RUSTSEC-2026-0058", pr_number: 24975, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 124, deletions_count: 166}, + {sha: "415059f727929adee4d88801ca760be2ac0b967a", date: "2026-03-23 23:19:24 UTC", description: "bump async-nats from 0.42.0 to 0.46.0", pr_number: 24974, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 6, deletions_count: 16}, + {sha: "05f7b4338b06ef7c8731cd8f391ca4e62f875b43", date: "2026-03-23 20:45:40 UTC", description: "add missing Windows source files to integration_windows filter", pr_number: 24992, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "f62a9ffa8729fb1e7e6d074d4d115c96f7f2740a", date: "2026-03-23 21:25:16 UTC", description: "ignore RUSTSEC-2026-0049 (rustls-webpki) until rustls can be upgraded", pr_number: 24986, scopes: ["deps"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "aedd2a66cb27b26621fff2c6d7dbd53a199fe0fa", date: "2026-03-23 21:26:02 UTC", description: "add security warning to path template field", pr_number: 24983, scopes: ["file sink"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 4, deletions_count: 0}, + {sha: "93a9771b1824c848c8efdf7166638226f01f42d2", date: "2026-03-24 14:22:00 UTC", description: "replace GraphQL observability API with gRPC, remove async-graphql dependencies", pr_number: 24364, scopes: ["api"], type: "chore", breaking_change: true, author: "Pavlos Rontidis", files_count: 128, insertions_count: 3020, deletions_count: 16042}, + {sha: "f1d8a93402168a2c5daf1eab2f4584aa5762bac4", date: "2026-03-25 00:52:15 UTC", description: "correct stop/start state checks and add configurable stop timeout", pr_number: 24772, scopes: ["windows platform"], type: "fix", breaking_change: false, author: "iMithrellas", files_count: 3, insertions_count: 100, deletions_count: 32}, + {sha: "0122cf6ec8fe7cce20a2bb694db137e3d5a9038a", date: "2026-03-24 20:04:14 UTC", description: "remove deprecated `headers` option from http and opentelemetry sinks", pr_number: 24994, scopes: ["sinks"], type: "chore", breaking_change: true, author: "Thomas", files_count: 9, insertions_count: 5, deletions_count: 43}, + {sha: "8bac1dbdcd27550444ea3b73e9dd4ef36172b2d4", date: "2026-03-24 23:00:40 UTC", description: "switch series v2 and sketches to zstd compression", pr_number: 24956, scopes: ["datadog_metrics sink"], type: "chore", breaking_change: false, author: "Vladimir Zhuk", files_count: 8, insertions_count: 629, deletions_count: 277}, + {sha: "753cf9a7c03a5cef85a29f3561666d94e9084d8b", date: "2026-03-24 23:57:50 UTC", description: "consolidate reference documentation into detailed docs table in AGENTS.md", pr_number: 25036, scopes: ["internal docs"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 15, deletions_count: 22}, + {sha: "47b5b02b9fdd13b67294df65175e9eca99c4cd8b", date: "2026-03-25 00:36:30 UTC", description: "split deny check into optional (all) and required (licenses)", pr_number: 24990, scopes: ["ci"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 38, deletions_count: 6}, + {sha: "ca9b29bc52b7c7ba857c337b25278edbad1c5e66", date: "2026-03-25 21:17:07 UTC", description: "bump serial_test from 3.2.0 to 3.4.0", pr_number: 25022, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 7, deletions_count: 6}, + {sha: "a2918e6ac4b874dc7c67b558a1e2bc9fd2b4ae81", date: "2026-03-25 21:58:38 UTC", description: "bump quick-junit from 0.5.2 to 0.6.0", pr_number: 25032, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "209b252deed214144fd2a6d4ed6c5ea3a3e08821", date: "2026-03-25 23:11:04 UTC", description: "enforce and fix markdownlint rules", pr_number: 25038, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 82, insertions_count: 415, deletions_count: 407}, + {sha: "7dc3ce3f34de6b79c0f09b4e44fe0e9d87cc09c2", date: "2026-03-26 02:32:34 UTC", description: "add links to design issues", pr_number: 25046, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 3, deletions_count: 0}, + {sha: "4fba57e9c4d6561c126851df8c4065221b44e992", date: "2026-03-26 07:51:02 UTC", description: "allow to add headers to metadata for OpenTelemetry metrics and traces", pr_number: 24942, scopes: ["opentelemetry source"], type: "feat", breaking_change: false, author: "Oleksandr Zanichkovskyi", files_count: 6, insertions_count: 463, deletions_count: 47}, + {sha: "498d1daa38432a329d168a2eec97a212922e71bc", date: "2026-03-26 04:51:47 UTC", description: "improve CLA instructions", pr_number: 25039, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 12, deletions_count: 13}, + {sha: "bc1e6bdde6b07f51efd4764c65d75c32d056499c", date: "2026-03-26 23:10:51 UTC", description: "bump uuid from 1.18.1 to 1.22.0", pr_number: 25025, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 37, deletions_count: 15}, + {sha: "072f6284c83279cd920177b32114f10f8fd5164c", date: "2026-03-26 23:15:29 UTC", description: "bump snafu from 0.8.9 to 0.9.0", pr_number: 25029, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 34, deletions_count: 13}, + {sha: "041769359ba0a9d68017e10e6e84cb7a83b3df27", date: "2026-03-26 23:22:32 UTC", description: "bump picomatch from 2.3.1 to 2.3.2 in /website", pr_number: 25047, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "d0c6cea89661577079d3eec790e4c054772de7d6", date: "2026-03-26 23:25:49 UTC", description: "bump yaml from 1.10.2 to 1.10.3 in /website", pr_number: 25043, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "5fcf1516151f659b5832fc433a4e10a219e31c7b", date: "2026-03-26 23:26:25 UTC", description: "bump ipnet from 2.11.0 to 2.12.0", pr_number: 25030, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "09629f02d60614a38b20b49cb10acbce47a9f6cc", date: "2026-03-26 23:49:33 UTC", description: "bump docker/setup-buildx-action from 3.12.0 to 4.0.0", pr_number: 25003, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 4, deletions_count: 4}, + {sha: "3255b66b2e8a2fb7ec3412904b856a8349a236ab", date: "2026-03-26 23:49:57 UTC", description: "bump actions/cache from 5.0.3 to 5.0.4", pr_number: 25002, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "9337c80bb9d2491a8d617df408c20a88385d719a", date: "2026-03-26 23:50:14 UTC", description: "bump docker/login-action from 3.7.0 to 4.0.0", pr_number: 25001, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 6, deletions_count: 6}, + {sha: "84abc68df859d58b2a7681757e52aca7a8094ad2", date: "2026-03-27 04:24:22 UTC", description: "bump bollard from 0.19.2 to 0.20.2", pr_number: 25026, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 9, deletions_count: 13}, + {sha: "6267fcc78d8cd347f6405ab03ca100b97944a5ef", date: "2026-03-27 06:31:32 UTC", description: "log error", pr_number: 24708, scopes: ["opentelemetry source"], type: "fix", breaking_change: false, author: "bas smit", files_count: 2, insertions_count: 10, deletions_count: 2}, + {sha: "e8526b2a3ef7ac257d0fb816f611cddbc494a478", date: "2026-03-27 06:32:43 UTC", description: "bump deadpool from 0.12.2 to 0.13.0", pr_number: 25014, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 23, deletions_count: 6}, + {sha: "af15a5a595f342038fd3368854fa75c74c31e932", date: "2026-03-27 05:09:12 UTC", description: "vanila `cargo build` should run on windows", pr_number: 24991, scopes: ["dev"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 13, deletions_count: 7}, + {sha: "4e6b49d6951b1f6ed9e2851aec674577ccc4c2b2", date: "2026-03-27 20:47:55 UTC", description: "deduct downstream utilization on task transforms", pr_number: 24731, scopes: ["metrics"], type: "fix", breaking_change: false, author: "Yoenn Burban", files_count: 3, insertions_count: 221, deletions_count: 18}, + {sha: "a7450cdbba3eb86782d67bea39b81e5b641067d9", date: "2026-03-27 18:50:50 UTC", description: "bump tempfile from 3.23.0 to 3.27.0", pr_number: 25016, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 14, deletions_count: 14}, + {sha: "7066adccecc24fe005f1fb7c934bbda4b557ea6c", date: "2026-03-27 18:51:39 UTC", description: "bump proptest from 1.10.0 to 1.11.0", pr_number: 25021, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 5, insertions_count: 7, deletions_count: 7}, + {sha: "3ca3374c5b0359a3839f6c8091e2c3632faec719", date: "2026-03-27 23:02:57 UTC", description: "bump rust_decimal from 1.39.0 to 1.40.0", pr_number: 24799, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "c1f34ef803fe5365a5be54f46877590bc8c91ba6", date: "2026-03-28 00:44:05 UTC", description: "bump brace-expansion from 1.1.12 to 1.1.13 in /website", pr_number: 25056, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "d29a524ea05dcbb6f823121ffdff449ecd960673", date: "2026-03-28 00:49:17 UTC", description: "bump security-framework from 3.5.1 to 3.6.0", pr_number: 24763, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 7, deletions_count: 7}, + {sha: "89e5c343c631e36090d2543106b88685d7a1c908", date: "2026-03-28 00:55:57 UTC", description: "bump juliangruber/read-file-action from 1.1.7 to 1.1.8", pr_number: 25000, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "55be10e5860afd47656a67a08335bb17a1d641c8", date: "2026-03-30 20:14:44 UTC", description: "add missing `contents: read` permission to integration review jobs", pr_number: 25067, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "d906ba19487c0552be7557d49a3f1dbfcd2405d9", date: "2026-03-31 01:12:33 UTC", description: "upgrade rustls from 0.23.23 to 0.23.37", pr_number: 25075, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 25, deletions_count: 13}, + {sha: "6a50bd50c9d482511f5a2a21466a3427dd752263", date: "2026-03-31 01:27:02 UTC", description: "update security policy structure and content", pr_number: 25074, scopes: ["internal docs"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 43, deletions_count: 59}, + {sha: "122cca4678172bbc0c067c65fcc909c6b7aa9ce6", date: "2026-03-31 01:51:55 UTC", description: "update GHCR cleanup workflow", pr_number: 25037, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 18, deletions_count: 14}, + {sha: "87ed519cc40011224208914f1dd79b7240f7e1d7", date: "2026-03-31 17:36:52 UTC", description: "consolidate advisory ignores with issue links", pr_number: 25076, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 4, deletions_count: 12}, + {sha: "84c7af2264deacb1afc0529671d53c8c6792bf88", date: "2026-03-31 17:41:49 UTC", description: "install vdev via setup action in ci-integration-review workflow", pr_number: 25077, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 10, deletions_count: 4}, + {sha: "d732259aaa6b5e156b001672ccf0e59be5e84802", date: "2026-03-31 18:08:30 UTC", description: "pin GitHub actions to full sha", pr_number: 25080, scopes: ["ci"], type: "chore", breaking_change: false, author: "StepSecurity Bot", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "0943dd938f247b0a1963e6bcb1a464ae49392c5f", date: "2026-03-31 22:54:02 UTC", description: "migrate from markdownlint-cli to markdownlint-cli2", pr_number: 25081, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 6, insertions_count: 13, deletions_count: 19}, + {sha: "50345d5929ecb45d74e1355e612f044c24ff4a93", date: "2026-03-31 22:27:15 UTC", description: "skip test runner pull, datadog-ci install, and checkout when int tests won't run", pr_number: 25068, scopes: ["ci"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 80, deletions_count: 74}, + {sha: "24d8db42e01e28b590e5484fede9279908805901", date: "2026-04-01 02:04:57 UTC", description: "remove unused publish-homebrew workflow", pr_number: 25085, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 0, deletions_count: 36}, + {sha: "df00c7eec91ecb35eb0a8db0249701562de7dc87", date: "2026-04-01 23:37:50 UTC", description: "pin Pulsar integration test image to 4.1.3", pr_number: 25097, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 1}, + {sha: "639c924c733442bcc7443817d16ea0dd737ded54", date: "2026-04-02 04:24:01 UTC", description: "bump debian from `1d3c811` to `26f98cc` in /distribution/docker/debian in the docker-images group across 1 directory", pr_number: 24996, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "9dbdb0662b668867ac8db6e444ac710d4346a037", date: "2026-04-02 04:25:08 UTC", description: "bump distroless/static from `28efbe9` to `47b2d72` in /distribution/docker/distroless-static in the docker-images group across 1 directory", pr_number: 24998, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "3c879836d586695701e9cf60fed3e7ef4b408f2e", date: "2026-04-02 04:25:30 UTC", description: "bump debian from `1d3c811` to `26f98cc` in /distribution/docker/distroless-libc in the docker-images group across 1 directory", pr_number: 24997, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "9f484c2b9c1c219f44c2fd5289c40e3a4b63f6e3", date: "2026-04-02 03:26:12 UTC", description: "prevent script injection in integration review workflow", pr_number: 25106, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 18, deletions_count: 2}, + {sha: "9055388bc0c89a56177f6df60de7a7e09fe74de1", date: "2026-04-02 17:48:09 UTC", description: "add repo-wide prettier config for YAML, JS, TS, and JSON", pr_number: 25082, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 226, insertions_count: 4028, deletions_count: 2779}, + {sha: "b59c9a25b899223b6633776eeafe2e67fe418640", date: "2026-04-02 19:32:25 UTC", description: "Add http_server source to semantic PR scope list", pr_number: 25078, scopes: ["ci"], type: "chore", breaking_change: false, author: "steveduan-IDME", files_count: 1, insertions_count: 1, deletions_count: 0}, + {sha: "1a78c93520ab25377b7162c9aa5756f29492f433", date: "2026-04-02 21:57:21 UTC", description: "fix Pulsar TLS integration tests with latest image", pr_number: 25099, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 5, deletions_count: 3}, + {sha: "dedb9667511691731c30c05235c319867d96580c", date: "2026-04-03 19:19:53 UTC", description: "pin npm transitive deps via package-lock.json and npm ci", pr_number: 25115, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 3449, deletions_count: 24}, + {sha: "04f3b782e95f044fc7307fe4f6230dc48793800b", date: "2026-04-03 18:23:45 UTC", description: "Reverse performance regression in buffer metrics", pr_number: 24995, scopes: ["buffers"], type: "fix", breaking_change: false, author: "Bruce Guenter", files_count: 1, insertions_count: 215, deletions_count: 114}, + {sha: "51030d2aade15deae23297a1ed1994cb36c530e9", date: "2026-04-03 20:36:36 UTC", description: "make file source tests not flaky", pr_number: 24957, scopes: ["dev"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 17, deletions_count: 6}, + {sha: "2fe515ccd2dfc23d98b84300e4fd87bee27b35fa", date: "2026-04-03 22:13:12 UTC", description: "use helm-charts develop branch for K8s E2E tests", pr_number: 25118, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 7, insertions_count: 44, deletions_count: 24}, + {sha: "b428aba5dd4f26285ce907212acae648bdb6d8df", date: "2026-04-03 23:39:14 UTC", description: "resolve build.rs HEAD path in worktrees", pr_number: 25120, scopes: ["dev"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 18, deletions_count: 1}, + {sha: "738c1d7758024a074f4fbe9ba9fa6d6110b316d7", date: "2026-04-04 05:14:48 UTC", description: "run K8s E2E suite on a weekly schedule", pr_number: 25119, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "703590bff3c2e49aa722c493329c753b1943c4ff", date: "2026-04-06 17:25:55 UTC", description: "Migrate smp regression workflow auth from static secrets to oidc", pr_number: 25112, scopes: ["ci"], type: "chore", breaking_change: false, author: "Caleb Metz", files_count: 1, insertions_count: 27, deletions_count: 12}, + {sha: "2d6fea2dfe7536202c83305cff5021bfadc90442", date: "2026-04-07 09:29:11 UTC", description: "update check-spelling", pr_number: 25124, scopes: ["ci"], type: "chore", breaking_change: false, author: "Jed Laundry", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "1152614bc73e74df27986d1bff7864a376025544", date: "2026-04-06 17:32:37 UTC", description: "collect K8s diagnostics on E2E test failure", pr_number: 25114, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 32, deletions_count: 1}, + {sha: "cab8fab16b2da5478d495cd37091b525f527d8ba", date: "2026-04-06 23:37:04 UTC", description: "bump the artifact group across 1 directory with 2 updates", pr_number: 24999, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 6, insertions_count: 17, deletions_count: 17}, + {sha: "4b2353a35dd54b888cdee7d1ca70c8723e12873c", date: "2026-04-06 23:46:52 UTC", description: "bump nick-fields/retry from 3.0.2 to 4.0.0", pr_number: 25102, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 5, insertions_count: 7, deletions_count: 7}, + {sha: "9e897388fbf772a23c48df93de923a05b2d3ecbe", date: "2026-04-07 04:05:11 UTC", description: "bump fast-xml-parser from 5.3.7 to 5.5.9 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory", pr_number: 25127, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 113, deletions_count: 81}, + {sha: "0c5fd58228bfab0dd9f6e81a1dc651da2ea8e670", date: "2026-04-07 20:14:58 UTC", description: "improve dashboard column layout", pr_number: 25135, scopes: ["api top"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 8, deletions_count: 8}, + {sha: "02f281d6ad2786f8bbfad9ece820b8d9f4925571", date: "2026-04-07 23:38:29 UTC", description: "show \"disabled\" in Memory Used when allocation tracing is off", pr_number: 25138, scopes: ["api top"], type: "enhancement", breaking_change: false, author: "Pavlos Rontidis", files_count: 7, insertions_count: 65, deletions_count: 2}, + {sha: "256db9c12951987ea235a0c5a4a88898d5187642", date: "2026-04-08 19:32:50 UTC", description: "avoid unused-mut warning on Windows builds", pr_number: 25142, scopes: ["api top"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 4, deletions_count: 3}, + {sha: "711b039fe7a8ddac2296dfe1e78685bb901795e6", date: "2026-04-08 21:30:51 UTC", description: "bump lodash from 4.17.23 to 4.18.1 in /website", pr_number: 25113, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "dcf98a7ccf54080a74626e9277ac5b2f8340ded8", date: "2026-04-08 21:37:57 UTC", description: "fix spell check CI failures", pr_number: 25144, scopes: [], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 3, deletions_count: 3}, + {sha: "30d9a58669556a9bc014ba0b2ea4f33f8e372d98", date: "2026-04-09 13:43:50 UTC", description: "Expand support for Azure authentication types", pr_number: 24729, scopes: ["azure_blob sink"], type: "feat", breaking_change: false, author: "Jed Laundry", files_count: 28, insertions_count: 1542, deletions_count: 342}, + {sha: "985f669ffae07c1104ae9cec1b097e74130e0341", date: "2026-04-09 09:50:38 UTC", description: "high CPU usage after async file server migration", pr_number: 25064, scopes: ["file source"], type: "fix", breaking_change: false, author: "fcfangcc", files_count: 5, insertions_count: 212, deletions_count: 11}, + {sha: "9c4abf31a91d888d0b4ca204643b0e6e1a559466", date: "2026-04-09 05:26:19 UTC", description: "emit VectorStopped after topology drains", pr_number: 25083, scopes: ["shutdown"], type: "fix", breaking_change: false, author: "tronboto", files_count: 3, insertions_count: 32, deletions_count: 10}, + {sha: "26eef13087e1a463efa4bdcc9d3919b13936e06c", date: "2026-04-09 00:26:27 UTC", description: "preserve `device` tag from v2 series resources", pr_number: 25146, scopes: ["datadog_agent source"], type: "fix", breaking_change: false, author: "Lisa Vu", files_count: 3, insertions_count: 69, deletions_count: 0}, + {sha: "b57d8b0ef0ec12a1645b556a6b1bf080dc66e50b", date: "2026-04-09 04:53:19 UTC", description: "bump basic-ftp from 5.2.0 to 5.2.1 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory", pr_number: 25147, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "51f6fce6d850dc4c6833d910013a189119068734", date: "2026-04-09 21:35:17 UTC", description: "fix typo in 24532 changelog fragment", pr_number: 25155, scopes: ["shutdown"], type: "chore", breaking_change: false, author: "Jonathan Davies", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "a7f6487c94b98766bdff802aac0cd1372b0311f6", date: "2026-04-13 21:07:40 UTC", description: "bump basic-ftp from 5.2.1 to 5.2.2 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory", pr_number: 25170, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "5ab0ba0d27392c70995a3d5e182a8307e65c641f", date: "2026-04-13 18:37:29 UTC", description: "bump axios from 1.13.5 to 1.15.0 in /website", pr_number: 25172, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 8, deletions_count: 8}, + {sha: "480ca58f52a019454a36e9c9b754791224fcee57", date: "2026-04-13 19:22:13 UTC", description: "extract vector-vrl-doc-builder into unpublished crate", pr_number: 25034, scopes: ["vdev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 10, insertions_count: 114, deletions_count: 58}, + {sha: "3112033384d47b0eae9fa36a1297dab32cac1f88", date: "2026-04-13 20:08:25 UTC", description: "convert TOML config examples to YAML", pr_number: 25163, scopes: ["website"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 68, insertions_count: 2096, deletions_count: 1829}, + {sha: "43f620435a353b048c40661f59a2d2d18f2419ed", date: "2026-04-13 20:24:37 UTC", description: "fix panic in allocation tracing dealloc path", pr_number: 25136, scopes: ["observability"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 192, deletions_count: 13}, + {sha: "c6574fd66c328ef57fba27d2ad5422666c3e18eb", date: "2026-04-13 20:26:51 UTC", description: "replace custom Health RPC with standard gRPC health service", pr_number: 25139, scopes: ["api"], type: "enhancement", breaking_change: true, author: "Pavlos Rontidis", files_count: 16, insertions_count: 119, deletions_count: 77}, + {sha: "44637c3056c8850752e6f1c6aef8ab5ead810b0f", date: "2026-04-14 19:04:29 UTC", description: "bump datadog-ci from 5.12.0 to 5.13.0", pr_number: 25187, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 145, deletions_count: 2278}, + {sha: "16458a84ba60660bbdc4258e3245981102da2921", date: "2026-04-14 19:48:09 UTC", description: "bump follow-redirects from 1.15.11 to 1.16.0 in /website", pr_number: 25189, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "e32661adb9138ff721d882f059f35eaf792d355c", date: "2026-04-15 02:53:04 UTC", description: "add Parquet encoder with schema_file and auto infer schema support", pr_number: 25156, scopes: ["aws_s3 sink"], type: "feat", breaking_change: false, author: "Peter Ehikhuemen", files_count: 21, insertions_count: 1818, deletions_count: 43}, + {sha: "51121047d3201394c2958e4a66696f3aef2d3f11", date: "2026-04-15 21:25:02 UTC", description: "bump rsa from 0.9.3 to 0.9.10", pr_number: 25198, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 10, deletions_count: 9}, + {sha: "dac7c31501defdf451f9422b6a569358fb5ec19e", date: "2026-04-15 21:53:42 UTC", description: "bump rustls-webpki 0.103.10 to 0.103.12 (RUSTSEC-2026-0099)", pr_number: 25200, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 5, deletions_count: 3}, + {sha: "6818b9c348cd4c3edd39f9f303f66362676e47a3", date: "2026-04-15 22:22:40 UTC", description: "add work-in-progress label workflow for docs PRs", pr_number: 24950, scopes: ["ci"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 68, deletions_count: 0}, + {sha: "2d14034bf8115764c52636a59ec298d7444a4d28", date: "2026-04-15 22:52:21 UTC", description: "override smol-toml to 1.6.1 for markdownlint-cli2", pr_number: 25202, scopes: ["deps"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 8, deletions_count: 3}, + {sha: "08c9f1e6b54cf1b30e88d536f3545e74ff193c0f", date: "2026-04-15 23:48:46 UTC", description: "bump VRL and add feature for enable_crypto_functions", pr_number: 25205, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 2, deletions_count: 1}, + {sha: "a1469590bed6996b1eb7001248232b0aa48cd777", date: "2026-04-16 00:06:08 UTC", description: "bump rand 0.10.0 to 0.10.1 and 0.9.2 to 0.9.4", pr_number: 25204, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 33, deletions_count: 32}, + {sha: "5ba8405bf48cd64509bf02388c7901451bbe67f4", date: "2026-04-16 00:20:05 UTC", description: "bump version to 0.3.1", pr_number: 25206, scopes: ["vdev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 2, deletions_count: 2}, + {sha: "ce651754d4e2eb134ee9ab2fe85a8a3a5ed923a0", date: "2026-04-16 17:16:53 UTC", description: "add source latency metric and fix source lag time on large batches", pr_number: 24987, scopes: ["sources"], type: "feat", breaking_change: false, author: "Yoenn Burban", files_count: 7, insertions_count: 122, deletions_count: 28}, + {sha: "8138c4593455dd284475ed4ed61fe55ee176bd34", date: "2026-04-16 19:27:11 UTC", description: "update DD RUM/Logs config to cose site", pr_number: 25213, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 9, deletions_count: 2}, + {sha: "61b4563fc60211feba19dcbf9296872b8640ec94", date: "2026-04-16 20:19:11 UTC", description: "fix secrets in static analysis workflow", pr_number: 25208, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 6, deletions_count: 3}, + {sha: "467ef876a286df316ba273e4ad1779f1b08ba392", date: "2026-04-17 03:18:13 UTC", description: "preserve site prefix when computing API endpoint", pr_number: 25211, scopes: ["datadog_common sink"], type: "fix", breaking_change: false, author: "Vladimir Zhuk", files_count: 2, insertions_count: 32, deletions_count: 1}, + {sha: "9efe09d254fdad82f5d1239e3eae28348390ddfa", date: "2026-04-17 04:34:41 UTC", description: "modernize cross ubuntu bootstrap script", pr_number: 25215, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 43, deletions_count: 24}, + {sha: "8790fa07177e93242395746b8328e2cd2e88b440", date: "2026-04-17 21:32:34 UTC", description: "split VERSION-dependent packaging targets into Makefile.packaging", pr_number: 25101, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 135, deletions_count: 108}, + {sha: "35351b9cc9ae7dffdf1a461203137905f987a8b5", date: "2026-04-18 01:22:12 UTC", description: "replace native encoding fixture patch files with cfg-gated code", pr_number: 24971, scopes: ["codecs"], type: "chore", breaking_change: false, author: "Thomas", files_count: 14, insertions_count: 130, deletions_count: 278}, + {sha: "604866d560deea7a14cbfa767a8ca99eee07089a", date: "2026-04-20 22:58:40 UTC", description: "use parent site for RUM to resolve intake host", pr_number: 25224, scopes: ["website"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "bdcf7476c55141ecf535cec6caafea7b2cc3d16d", date: "2026-04-20 23:43:03 UTC", description: "bump dorny/paths-filter from 3.0.2 to 4.0.1", pr_number: 25105, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 6, deletions_count: 6}, + {sha: "c2022541ee792988c465ca9a76b93a0f54ea4e00", date: "2026-04-20 23:43:44 UTC", description: "add greptimedb v0 support deprecation entry", pr_number: 25226, scopes: ["deprecations"], type: "docs", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 2, deletions_count: 0}, + {sha: "0f7574ad33ce07243ab8f9d6399c5e6c2f451af1", date: "2026-04-21 00:49:34 UTC", description: "fix panic in allocation tracing dealloc path", pr_number: 25222, scopes: ["observability"], type: "revert", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 13, deletions_count: 192}, + {sha: "3ca3e61ec8b8cd89be836b47346343069b229398", date: "2026-04-21 01:11:31 UTC", description: "use CUE raw multi-line strings in release generator", pr_number: 25228, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 5, deletions_count: 2}, + ] +} diff --git a/website/cue/reference/versions.cue b/website/cue/reference/versions.cue index ae51453c8f728..0ca3310065d7f 100644 --- a/website/cue/reference/versions.cue +++ b/website/cue/reference/versions.cue @@ -2,6 +2,7 @@ package metadata // This has to be maintained manually because there's currently no way to sort versions programmatically versions: [string, ...string] & [ + "0.55.0", "0.54.0", "0.53.0", "0.52.0", diff --git a/website/layouts/releases/single.html b/website/layouts/releases/single.html index fe4cb606aacdf..f83cb9e349122 100644 --- a/website/layouts/releases/single.html +++ b/website/layouts/releases/single.html @@ -31,6 +31,14 @@

{{ end }} + {{ with $highlights }} +
+ {{ range . }} + {{ .Render "li" }} + {{ end }} +
+ {{ end }} + {{ with $release.description }}
{{ . | markdownify }} @@ -60,20 +68,6 @@
- {{ with $highlights }} -
-
- {{ partial "heading.html" (dict "text" "Highlights" "level" 2) }} -
- -
- {{ range . }} - {{ .Render "li" }} - {{ end }} -
-
- {{ end }} - {{ with $release.known_issues }} {{ if gt (len $release.known_issues) 0 }}
From f36ed0e2cebf36efdc7392aed0ed52ae28a29101 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Apr 2026 15:54:48 -0400 Subject: [PATCH 101/364] chore(website): render release date on per-version release page (#25244) Mirror the pattern used by the highlights/guides content header and render \$release.date below the hero on /releases// pages. The date already exists in each version's CUE (e.g. date: "2026-04-22" in website/cue/reference/releases/0.55.0.cue) but was only surfaced on the releases index (releases/li.html). Co-authored-by: Claude Opus 4.7 (1M context) --- website/layouts/releases/single.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/website/layouts/releases/single.html b/website/layouts/releases/single.html index f83cb9e349122..c1708ea955a9c 100644 --- a/website/layouts/releases/single.html +++ b/website/layouts/releases/single.html @@ -18,6 +18,15 @@ {{ partial "hero.html" . }}
+ {{ with $release.date }} + {{ $date := . | dateFormat "January 2, 2006" }} +
+ +
+ {{ end }} +
{{ with $release.codename }}

From 29b17aadc63fdd9a9bdc3b64bbd8f9c647844cdb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Apr 2026 16:30:30 -0400 Subject: [PATCH 102/364] fix(ci): skip re-adding WIP label on synchronize if already approved (#25246) The add_wip_label workflow fires on synchronize to catch docs paths added in later pushes. However, it also re-added the label after a reviewer had already approved and remove_wip_label removed it. Guard against that by checking the latest review state per member: skip only when at least one member approved and none have a standing changes-requested review. This preserves the "catch docs added later" behavior while avoiding label churn after approval. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/add_wip_label.yml | 30 ++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_wip_label.yml index a07f9ca0921fa..9a1260b693e81 100644 --- a/.github/workflows/add_wip_label.yml +++ b/.github/workflows/add_wip_label.yml @@ -17,6 +17,34 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 + - name: Check member review state + id: check + uses: actions/github-script@v7 + with: + script: | + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }); + + const allowed = ['MEMBER', 'OWNER']; + const latestByUser = new Map(); + for (const r of reviews) { + if (!allowed.includes(r.author_association)) continue; + if (r.state === 'COMMENTED') continue; + latestByUser.set(r.user.id, r.state); + } + + const states = [...latestByUser.values()]; + const hasApproved = states.includes('APPROVED'); + const hasChangesRequested = states.includes('CHANGES_REQUESTED'); + const skip = hasApproved && !hasChangesRequested; + + core.info(`Latest member review states: ${states.join(', ') || '(none)'}`); + core.info(`skip=${skip}`); + core.setOutput('skip', skip); + - if: steps.check.outputs.skip != 'true' + uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 with: labels: "work in progress" From d99899e2f9cb94c8d421270ada8688243402b5a1 Mon Sep 17 00:00:00 2001 From: "shalk(xiao kun)" Date: Thu, 23 Apr 2026 04:41:08 +0800 Subject: [PATCH 103/364] fix(website): fix docs sidebar expand/collapse navigation (#25238) - Use strings.HasPrefix to keep sections expanded when on any child page (the previous eq check only matched the section index, not child pages) - Add @click="open = true" to section title links so clicking the title expands the group in addition to navigating, without requiring the chevron Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Pavlos Rontidis --- website/layouts/partials/docs/sidebar.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/website/layouts/partials/docs/sidebar.html b/website/layouts/partials/docs/sidebar.html index f9c81ed763cd8..e9b9af98de037 100644 --- a/website/layouts/partials/docs/sidebar.html +++ b/website/layouts/partials/docs/sidebar.html @@ -47,10 +47,13 @@ {{ define "subsection-group" }} {{ $here := .here }} {{ $section := .section }} -{{ $open := or (.ctx.IsAncestor $section) (eq .ctx.RelPermalink $here) }} +{{ $open := or (.ctx.IsAncestor $section) (strings.HasPrefix $here .ctx.RelPermalink) }} +{{ $isActive := eq .ctx.RelPermalink $here }}

- {{ template "link" (dict "here" $here "url" .ctx.RelPermalink "title" (.ctx.Params.short | default .ctx.Title)) }} + + {{ .ctx.Params.short | default .ctx.Title }} + {{ template "chevron-icon" }} From ecdaa50ac15e6a26e811bf21e3a6e33729bebb33 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 22 Apr 2026 19:23:12 -0400 Subject: [PATCH 104/364] fix(ci): verify choco package install after 5xx from feed (#25116) * fix(ci): verify choco package install after 5xx from feed * fix(ci): match package name in choco list output to verify install --- scripts/environment/bootstrap-windows-2025.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/environment/bootstrap-windows-2025.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 index 862efc554a187..9686811a97238 100644 --- a/scripts/environment/bootstrap-windows-2025.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -11,7 +11,10 @@ function Install-ChocoPackage { for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { choco install $Package --execution-timeout=7200 -y - if ($LASTEXITCODE -eq 0) { + # Both `choco install` and `choco list` can exit 0 even on 5xx errors + # from the feed, so verify install by matching a "name|version" line + # in the list output. --limit-output strips headers/warnings. + if ((choco list --limit-output -e $Package) -match "^$([regex]::Escape($Package))\|") { return } From f1557d68c8e7f36feb8e2d0c7a04286817c6bf9f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 23 Apr 2026 10:16:39 -0400 Subject: [PATCH 105/364] feat(unit tests): add `expected_event_count` field to test outputs (#25186) * feat(unit tests): add `expected_event_count` field to test outputs Add an optional `expected_event_count` field to the `TestOutput` config struct, allowing users to assert on the number of events emitted by a transform in unit tests. The count check is independent of conditions and is verified first. - Reject `expected_event_count: 0` with conditions at build time - Error on conflicting counts when multiple outputs share extract_from - Allow `expected_event_count: 0` to suppress the "no events received" error when zero events are genuinely expected Closes #20278 Co-Authored-By: Claude Opus 4.6 (1M context) * fix(unit tests): add missing authors line to changelog fragment Co-Authored-By: Claude Opus 4.6 (1M context) * fix(unit tests): reject zero-count outputs with conditions after merge A config could bypass the zero-count/conditions validation by splitting expected_event_count: 0 and conditions across two output entries sharing the same extract_from. After merge the combined entry had count 0 with non-empty conditions, causing conditions to pass vacuously against zero events. Add a post-merge check to catch this. Co-Authored-By: Claude Opus 4.6 (1M context) * fix author * chore(unit tests): replace output tuple with BuiltOutput struct Addresses review feedback on PR #25186: the `(Option, Vec>)` tuple in `build_outputs` was too opaque, so this introduces a `BuiltOutput` struct with named fields. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(unit tests): fix indoc indentation in expected_event_count tests Addresses review nit on PR #25186: the YAML in `expected_event_count_correct` and `expected_event_count_wrong` was over-indented; align it with the other tests in the file. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .../20278_expected_event_count.feature.md | 3 + src/config/mod.rs | 16 +- src/config/unit_test/mod.rs | 58 ++++- src/config/unit_test/tests.rs | 235 ++++++++++++++++++ src/config/unit_test/unit_test_components.rs | 22 +- 5 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 changelog.d/20278_expected_event_count.feature.md diff --git a/changelog.d/20278_expected_event_count.feature.md b/changelog.d/20278_expected_event_count.feature.md new file mode 100644 index 0000000000000..23dac198ecdea --- /dev/null +++ b/changelog.d/20278_expected_event_count.feature.md @@ -0,0 +1,3 @@ +Unit tests now support an optional `expected_event_count` field on test outputs, allowing assertions on the number of events emitted by a transform. + +authors: pront diff --git a/src/config/mod.rs b/src/config/mod.rs index dceade72d6eeb..4a24954926c13 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -446,11 +446,12 @@ impl TestDefinition { let TestOutput { extract_from, conditions, + expected_event_count, } = old; - (extract_from.to_vec(), conditions) + (extract_from.to_vec(), conditions, expected_event_count) }) - .filter_map(|(extract_from, conditions)| { + .filter_map(|(extract_from, conditions, expected_event_count)| { let mut outputs = Vec::new(); for from in extract_from { if no_outputs_from.contains(&from) { @@ -471,6 +472,7 @@ impl TestDefinition { Some(TestOutput { extract_from: outputs.into(), conditions, + expected_event_count, }) } }) @@ -525,6 +527,7 @@ impl TestDefinition { .collect::>() .into(), conditions: old.conditions, + expected_event_count: old.expected_event_count, }) .collect(); @@ -596,6 +599,15 @@ pub struct TestOutput { /// The conditions to run against the output to validate that they were transformed as expected. pub conditions: Option>, + + /// The expected number of events to be produced by the transform. + /// + /// If specified, the test will fail if the number of events emitted by the + /// transform does not match this value. This check is independent of + /// `conditions` -- the count is verified first, then each condition is + /// evaluated against the output events separately. This is useful for + /// transforms that may emit multiple events. + pub expected_event_count: Option, } #[cfg(all(test, feature = "sources-file", feature = "sinks-console"))] diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index 4d8c1fa97ad65..5020c0b506629 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -293,14 +293,17 @@ impl UnitTestBuildMetadata { let mut template_sinks = IndexMap::new(); let mut test_result_rxs = Vec::new(); // Add sinks with checks - for (ids, checks) in outputs { + for (ids, built) in outputs { let (tx, rx) = oneshot::channel(); let sink_ids = ids.clone(); let sink_config = UnitTestSinkConfig { test_name: test_name.to_string(), transform_ids: ids.iter().map(|id| id.to_string()).collect(), result_tx: Arc::new(Mutex::new(Some(tx))), - check: UnitTestSinkCheck::Checks(checks), + check: UnitTestSinkCheck::Checks { + conditions: built.conditions, + expected_event_count: built.expected_event_count, + }, }; test_result_rxs.push(rx); @@ -567,10 +570,16 @@ fn build_and_validate_inputs( } } +#[derive(Default)] +pub(super) struct BuiltOutput { + pub(super) expected_event_count: Option, + pub(super) conditions: Vec>, +} + fn build_outputs( test_outputs: &[TestOutput], -) -> Result, Vec>>, Vec> { - let mut outputs: IndexMap, Vec>> = IndexMap::new(); +) -> Result, BuiltOutput>, Vec> { + let mut outputs: IndexMap, BuiltOutput> = IndexMap::new(); let mut errors = Vec::new(); for output in test_outputs { @@ -590,10 +599,47 @@ fn build_outputs( } } + let expected_event_count = output.expected_event_count; + if expected_event_count == Some(0) && !conditions.is_empty() { + errors.push(format!( + "output for {:?} has expected_event_count of 0 but also defines conditions; \ + conditions cannot be evaluated when no events are expected", + output.extract_from + )); + } outputs .entry(output.extract_from.clone().to_vec()) - .and_modify(|existing_conditions| existing_conditions.push(conditions.clone())) - .or_insert(vec![conditions.clone()]); + .and_modify(|existing| { + if let (Some(prev), Some(new)) = + (existing.expected_event_count, expected_event_count) + { + if prev != new { + errors.push(format!( + "conflicting expected_event_count for extract_from {:?}: {} vs {}", + output.extract_from, prev, new + )); + } + } else if existing.expected_event_count.is_none() { + existing.expected_event_count = expected_event_count; + } + existing.conditions.push(conditions.clone()); + }) + .or_insert_with(|| BuiltOutput { + expected_event_count, + conditions: vec![conditions.clone()], + }); + } + + // Post-merge validation: after merging entries that share the same + // extract_from, reject any that ended up with expected_event_count of 0 and + // non-empty conditions (which would pass vacuously against zero events). + for (extract_from, built) in &outputs { + if built.expected_event_count == Some(0) && built.conditions.iter().any(|c| !c.is_empty()) { + errors.push(format!( + "output for {extract_from:?} has expected_event_count of 0 but also defines conditions; \ + conditions cannot be evaluated when no events are expected", + )); + } } if errors.is_empty() { diff --git a/src/config/unit_test/tests.rs b/src/config/unit_test/tests.rs index 938286ac9ba9f..6881ba6f75901 100644 --- a/src/config/unit_test/tests.rs +++ b/src/config/unit_test/tests.rs @@ -1298,3 +1298,238 @@ async fn test_glob_input() { let mut tests = build_unit_tests(config).await.unwrap(); assert!(tests.remove(0).run().await.errors.is_empty()); } + +#[tokio::test] +async fn expected_event_count_correct() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}, {"message": "three"}] + tests: + - name: event count check + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 3 + conditions: + - type: vrl + source: | + assert!(exists(.message), "message field must exist") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + assert!(tests.remove(0).run().await.errors.is_empty()); +} + +#[tokio::test] +async fn expected_event_count_wrong() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}, {"message": "three"}] + tests: + - name: wrong event count + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 2 + conditions: + - type: vrl + source: | + assert!(exists(.message), "message field must exist") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + let errors = tests.remove(0).run().await.errors; + assert!(!errors.is_empty()); + assert!( + errors + .iter() + .any(|e| e.contains("expected 2 events") && e.contains("but received 3")), + "expected event count error, got: {errors:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_zero_no_conditions() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: zero events expected + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + assert!(tests.remove(0).run().await.errors.is_empty()); +} + +#[tokio::test] +async fn expected_event_count_zero_with_conditions_rejected() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: zero with conditions + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + conditions: + - type: vrl + source: | + assert!(exists(.message), "unreachable") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("expected_event_count of 0") && e.contains("conditions")), + "expected config error about zero count with conditions, got: {errs:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_conflicting_values() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}] + tests: + - name: conflicting counts + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 2 + conditions: + - type: vrl + source: | + assert!(exists(.message), "ok") + - extract_from: foo + expected_event_count: 5 + conditions: + - type: vrl + source: | + assert!(exists(.message), "ok") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("conflicting expected_event_count")), + "expected conflicting count error, got: {errs:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_zero_split_with_conditions_rejected() { + crate::test_util::trace_init(); + + // Splitting expected_event_count: 0 and conditions across two output + // entries that share the same extract_from must still be rejected after + // merge, otherwise the conditions would pass vacuously against zero events. + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: split zero with conditions + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + - extract_from: foo + conditions: + - type: vrl + source: | + assert!(false, "should never be evaluated") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("expected_event_count of 0") && e.contains("conditions")), + "expected config error about zero count with conditions after merge, got: {errs:?}" + ); +} diff --git a/src/config/unit_test/unit_test_components.rs b/src/config/unit_test/unit_test_components.rs index 02808fb083ee7..b3f09098237da 100644 --- a/src/config/unit_test/unit_test_components.rs +++ b/src/config/unit_test/unit_test_components.rs @@ -117,7 +117,10 @@ impl SourceConfig for UnitTestStreamSourceConfig { #[derive(Clone, Default)] pub enum UnitTestSinkCheck { /// Check all events that are received against the list of conditions. - Checks(Vec>), + Checks { + conditions: Vec>, + expected_event_count: Option, + }, /// Check that no events were received. NoOutputs, @@ -203,8 +206,21 @@ impl StreamSink for UnitTestSink { } match self.check { - UnitTestSinkCheck::Checks(checks) => { - if output_events.is_empty() { + UnitTestSinkCheck::Checks { + conditions: checks, + expected_event_count, + } => { + if let Some(expected) = expected_event_count { + let actual = output_events.len(); + if actual != expected { + result.test_errors.push(format!( + "expected {} events from transforms {:?}, but received {}", + expected, self.transform_ids, actual + )); + } + } + + if output_events.is_empty() && expected_event_count != Some(0) { result .test_errors .push(format!("checks for transforms {:?} failed: no events received. Topology may be disconnected or transform is missing inputs.", self.transform_ids)); From cfb942bf7d1983b9eeaeb93ba2f843a816d1db5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:39:22 -0400 Subject: [PATCH 106/364] chore(deps): bump openssl from 0.10.75 to 0.10.78 (#25250) Bumps [openssl](https://github.com/rust-openssl/rust-openssl) from 0.10.75 to 0.10.78. - [Release notes](https://github.com/rust-openssl/rust-openssl/releases) - [Commits](https://github.com/rust-openssl/rust-openssl/compare/openssl-v0.10.75...openssl-v0.10.78) --- updated-dependencies: - dependency-name: openssl dependency-version: 0.10.78 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- lib/vector-core/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a39a96a6500b1..57f9a916e8812 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7373,9 +7373,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" dependencies = [ "bitflags 2.10.0", "cfg-if", @@ -7414,9 +7414,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" dependencies = [ "cc", "libc", diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 7d2d8f8979266..c26953cb3590a 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -34,7 +34,7 @@ metrics-util.workspace = true mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } ordered-float.workspace = true -openssl = { version = "0.10.75", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.78", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.5", default-features = false } pin-project.workspace = true proptest = { workspace = true, optional = true } From c0cdf2068f79176fd8af4dca9b27cabae5be3091 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 23 Apr 2026 13:52:07 -0400 Subject: [PATCH 107/364] chore(ci): extract unit tests into reusable workflow (#25247) * chore(ci): extract unit tests into reusable workflow * chore(ci): fix component-validation input and add OS-based FEATURES default * chore(ci): pass Datadog credentials into reusable unit-tests workflow --- .github/workflows/changes.yml | 1 + .github/workflows/test.yml | 30 ++------ .github/workflows/unit-tests.yml | 118 +++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index f9cd9ce4ba5b9..b07447715d75f 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -280,6 +280,7 @@ jobs: - ".github/workflows/changes.yml" test-yml: - ".github/workflows/test.yml" + - ".github/workflows/unit-tests.yml" - ".github/workflows/changes.yml" integration-yml: - ".github/workflows/integration.yml" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 81ceef93ca5f1..84a4c31226b1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,31 +56,13 @@ jobs: - run: make check-clippy test: - name: Unit and Component Validation tests - x86_64-unknown-linux-gnu - runs-on: ubuntu-24.04-8core - if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: ./.github/actions/setup - with: - rust: true - cargo-nextest: true - datadog-ci: true - protoc: true - libsasl2: true - - name: Unit Test - run: make test - env: - CARGO_BUILD_JOBS: 5 - - # Validates components for adherence to the Component Specification - - name: Check Component Spec - run: make test-component-validation - - - name: Upload test results - run: scripts/upload-test-results.sh - if: always() + if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} + uses: ./.github/workflows/unit-tests.yml + with: + default: true + component-validation: true + secrets: inherit check-scripts: name: Check scripts diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000000..e3b6501247858 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,118 @@ +# Reusable Workflow: Tests +# +# Runs test suites via the Makefile. By default runs unit tests and component +# validation. Callers can enable additional suites (CLI, Vector API, behavior) +# via boolean inputs. +# +# When `default` is true, all nextest-based suites run in a single `make test` +# invocation with the relevant features enabled. When `default` is false, only +# the explicitly enabled suites run, using a nextest filter expression to +# select the matching tests. + +name: Tests + +on: + workflow_call: + inputs: + ref: + description: "Git ref to checkout" + required: false + type: string + default: + description: "Run the full default unit test suite (--workspace with default features)" + required: true + type: boolean + component-validation: + description: "Include component validation tests" + required: false + type: boolean + default: true + cli: + description: "Include CLI tests" + required: false + type: boolean + default: false + vector-api: + description: "Include Vector API tests" + required: false + type: boolean + default: false + behavior: + description: "Include behavior tests" + required: false + type: boolean + default: false + +permissions: + contents: read + +env: + CARGO_INCREMENTAL: "0" + CI: true + DD_ENV: "ci" + DD_API_KEY: ${{ secrets.DD_API_KEY }} + +jobs: + tests: + name: Tests + runs-on: ubuntu-24.04-8core + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} + + - uses: ./.github/actions/setup + with: + rust: true + cargo-nextest: true + datadog-ci: true + protoc: true + libsasl2: true + + - name: Run tests + run: | + add_suite() { + local enabled="$1" feature="$2" filter="$3" + [[ "${enabled}" != "true" ]] && return + FEATURES="${FEATURES:+$FEATURES,}${feature}" + if [[ "${{ inputs.default }}" != "true" ]]; then + SCOPE="${SCOPE:+$SCOPE | }${filter}" + fi + } + + # Mirror the Makefile's OS-based default: `default-msvc` on Windows, + # `default` elsewhere. Keep in sync with the FEATURES defaults in Makefile. + if [[ "${RUNNER_OS}" == "Windows" ]]; then + DEFAULT_FEATURES="default-msvc" + else + DEFAULT_FEATURES="default" + fi + + if [[ "${{ inputs.default }}" == "true" ]]; then + FEATURES="${DEFAULT_FEATURES}" + else + FEATURES="" + fi + + add_suite "${{ inputs.component-validation }}" "component-validation-tests" "test(components::validation::tests)" + add_suite "${{ inputs.cli }}" "cli-tests" "package(vector) & binary(=integration)" + add_suite "${{ inputs.vector-api }}" "vector-api-tests" "binary(=vector_api)" + + # Skip `make test` entirely if no nextest-based suite was selected + # (e.g. behavior-only runs). Running `make test` with empty FEATURES + # would otherwise execute the workspace-wide suite. + if [[ "${{ inputs.default }}" == "true" ]]; then + make test FEATURES="${FEATURES}" + elif [[ -n "${SCOPE}" ]]; then + make test FEATURES="${FEATURES}" SCOPE="-E '${SCOPE}'" + else + echo "No nextest suites selected; skipping 'make test'" + fi + + - name: Behavior tests + if: ${{ inputs.behavior }} + run: make test-behavior + + - name: Upload test results to Datadog + if: ${{ always() }} + run: scripts/upload-test-results.sh From f56f4fa7234440345d790a2b81c7aa880be831d8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 23 Apr 2026 16:29:50 -0400 Subject: [PATCH 108/364] chore(ci): remove Chocolatey from Windows bootstrap (#25254) * chore(ci): remove Chocolatey from Windows bootstrap Install protoc directly from the upstream GitHub release (matching the pinned version used on Linux/macOS in scripts/environment/install-protoc.sh) instead of going through Chocolatey. GNU make is already on the default PATH of the windows-2025 runner image via C:\mingw64\bin, so no extra install step is needed for it. This eliminates the recurring "Chocolatey installed 0/0 packages" failures caused by Chocolatey CDN blips, and pins the Windows build to the same protoc version used on Linux/macOS. Verified on the ci-sandbox repo: - protoc resolves to $RUNNER_TEMP\protoc\bin\protoc.exe -> libprotoc 3.21.12 - make resolves to C:\mingw64\bin\make.exe -> GNU Make 4.4.1 Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): share install-protoc.sh across platforms Extend the existing Linux/macOS protoc installer with MINGW/MSYS platform detection and .exe handling, and have bootstrap-windows-2025.ps1 delegate to it via Git Bash. This replaces the inline PowerShell download introduced earlier and makes all three platforms pin the same protoc version from the same upstream source. Also adds curl --retry 5 --retry-delay 10 --retry-all-errors so transient download blips are retried without extra scaffolding. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): drop curl --retry-all-errors for focal compatibility The ghcr.io/cross-rs/* base images used by scripts/cross/Dockerfile are Ubuntu 20.04 focal, which ships curl 7.68.x. --retry-all-errors was added in curl 7.71.0, so it would be rejected as an unknown option and break cross-target builds that run install-protoc.sh inside those images. Plain --retry 5 --retry-delay 10 still handles the transport-level retries that cover the CDN blips we care about. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../environment/bootstrap-windows-2025.ps1 | 42 ++++++------------- scripts/environment/install-protoc.sh | 37 +++++++++++----- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/scripts/environment/bootstrap-windows-2025.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 index 9686811a97238..7d73eeccbc32d 100644 --- a/scripts/environment/bootstrap-windows-2025.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -2,32 +2,6 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version Latest -# Helper function to install choco packages with exponential backoff retry -function Install-ChocoPackage { - param( - [string]$Package, - [int]$MaxRetries = 5 - ) - - for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { - choco install $Package --execution-timeout=7200 -y - # Both `choco install` and `choco list` can exit 0 even on 5xx errors - # from the feed, so verify install by matching a "name|version" line - # in the list output. --limit-output strips headers/warnings. - if ((choco list --limit-output -e $Package) -match "^$([regex]::Escape($Package))\|") { - return - } - - if ($attempt -lt $MaxRetries) { - $delay = 5 * [math]::Pow(2, $attempt) # Exponential: 10, 20, 40, 80 seconds - Write-Host "choco install $Package failed (attempt $attempt of $MaxRetries). Retrying in $delay seconds..." - Start-Sleep -Seconds $delay - } else { - throw "choco install $Package failed after $MaxRetries attempts" - } - } -} - # Set up our Cargo path so we can do Rust-y things. echo "$HOME\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append @@ -37,9 +11,19 @@ if ($env:RELEASE_BUILDER -ne "true") { bash scripts/environment/prepare.sh --modules=rustup } -# Install Chocolatey packages with exponential backoff retry -Install-ChocoPackage "make" -Install-ChocoPackage "protoc" +# Install protoc via the shared cross-platform script. It pins the same version +# used on Linux/macOS and downloads directly from the upstream GitHub release, +# so we avoid the recurring Chocolatey CDN failures. +$ProtocInstallDir = Join-Path $env:RUNNER_TEMP "protoc-bin" +bash scripts/environment/install-protoc.sh "$ProtocInstallDir" +if ($LASTEXITCODE -ne 0) { + throw "install-protoc.sh failed with exit code $LASTEXITCODE" +} +echo "$ProtocInstallDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + +# GNU make is already on PATH on the windows-2025 runner image via the +# pre-installed MinGW toolchain at C:\mingw64\bin, so no extra install is +# needed here. # Set a specific override path for libclang. echo "LIBCLANG_PATH=$( (gcm clang).source -replace "clang.exe" )" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/scripts/environment/install-protoc.sh b/scripts/environment/install-protoc.sh index 4d1debe6c3251..1e75279a3aec2 100755 --- a/scripts/environment/install-protoc.sh +++ b/scripts/environment/install-protoc.sh @@ -23,13 +23,12 @@ trap 'rm -rf "${TMP_DIR?}"' EXIT get_platform() { local os os=$(uname) - if [[ "${os}" == "Darwin" ]]; then - echo "osx" - elif [[ "${os}" == "Linux" ]]; then - echo "linux" - else - >&2 echo "unsupported os: ${os}" && exit 1 - fi + case "${os}" in + Darwin) echo "osx" ;; + Linux) echo "linux" ;; + MINGW*|MSYS*|CYGWIN*) echo "win64" ;; + *) >&2 echo "unsupported os: ${os}" && exit 1 ;; + esac } get_arch() { @@ -47,20 +46,36 @@ get_arch() { fi } +get_bin_name() { + if [[ "$(get_platform)" == "win64" ]]; then + echo "protoc.exe" + else + echo "protoc" + fi +} + install_protoc() { local version=$1 local install_path=$2 local base_url="https://github.com/protocolbuffers/protobuf/releases/download" local url - url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + if [[ "$(get_platform)" == "win64" ]]; then + # Windows release assets are named without an explicit arch suffix. + url="${base_url}/v${version}/protoc-${version}-win64.zip" + else + url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + fi local download_path="${TMP_DIR}/protoc.zip" echo "Downloading ${url}" - curl -fsSL "${url}" -o "${download_path}" + # Stay compatible with the curl shipped by Ubuntu 20.04 focal (7.68.x) used + # in the ghcr.io/cross-rs/* base images: --retry-all-errors was only added + # in curl 7.71.0, so rely on the transport-level retry that --retry covers. + curl --retry 5 --retry-delay 10 -fsSL "${url}" -o "${download_path}" unzip -qq "${download_path}" -d "${TMP_DIR}" - mv -f -v "${TMP_DIR}/bin/protoc" "${install_path}" + mv -f -v "${TMP_DIR}/bin/$(get_bin_name)" "${install_path}" } -install_protoc "21.12" "${INSTALL_PATH}/protoc" +install_protoc "21.12" "${INSTALL_PATH}/$(get_bin_name)" From 7428dc3f4b23c56f1f2eecf44b3f6b2532c28dac Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Apr 2026 14:40:53 -0400 Subject: [PATCH 109/364] chore(ci): use dd-sts instead of DD_API_KEY (#25235) * chore(ci): use dd-octo-sts instead of DD_API_KEY * chore(ci): rename policy to public-vectordotdev-vector * chore(ci): grant id-token: write to reusable workflow callers * Format * chore(ci): introduce dd-token composite action to federate Datadog credentials * Fix policy name --- .github/actions/dd-token/action.yml | 28 +++++++++++++++++++++ .github/workflows/ci-integration-review.yml | 9 ++++++- .github/workflows/ci-review-trigger.yml | 1 + .github/workflows/coverage.yml | 6 ++++- .github/workflows/integration-test.yml | 6 ++++- .github/workflows/integration.yml | 13 +++++++++- .github/workflows/master_merge_queue.yml | 2 +- .github/workflows/static-analysis.yml | 12 ++++++--- .github/workflows/test-make-command.yml | 7 +++++- .github/workflows/test.yml | 4 ++- .github/workflows/unit-tests.yml | 6 ++++- 11 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 .github/actions/dd-token/action.yml diff --git a/.github/actions/dd-token/action.yml b/.github/actions/dd-token/action.yml new file mode 100644 index 0000000000000..8ea92dc1c2dd5 --- /dev/null +++ b/.github/actions/dd-token/action.yml @@ -0,0 +1,28 @@ +name: "Datadog Short-Lived Token" +description: "Federate a short-lived Datadog token via dd-sts-action and export DD_API_KEY (and DD_APP_KEY when available) to the job environment." + +inputs: + policy: + required: false + default: public-vectordotdev-vector + description: "dd-sts policy to federate against." + +runs: + using: "composite" + steps: + - name: Federate Datadog token + id: dd-sts + uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 + with: + policy: ${{ inputs.policy }} + + - name: Export Datadog credentials to environment + shell: bash + env: + DD_STS_API_KEY: ${{ steps.dd-sts.outputs.api_key }} + DD_STS_APP_KEY: ${{ steps.dd-sts.outputs.app_key }} + run: | + echo "DD_API_KEY=${DD_STS_API_KEY}" >> "$GITHUB_ENV" + if [ -n "${DD_STS_APP_KEY}" ]; then + echo "DD_APP_KEY=${DD_STS_APP_KEY}" >> "$GITHUB_ENV" + fi diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 77b20b347e318..b042a76eb8ba9 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -41,7 +41,6 @@ env: TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true @@ -100,6 +99,7 @@ jobs: timeout-minutes: 90 permissions: contents: read + id-token: write packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false @@ -160,6 +160,9 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/dd-token + if: steps.run_condition.outputs.should_run == 'true' + - uses: ./.github/actions/setup with: vdev: true @@ -191,6 +194,7 @@ jobs: permissions: contents: read packages: read # Required to pull test runner image from GHCR + id-token: write strategy: fail-fast: false matrix: @@ -214,6 +218,9 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/dd-token + if: steps.run_condition.outputs.should_run == 'true' + - uses: ./.github/actions/setup with: vdev: true diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index 8a2eb206efdcd..9cd2a1d67e3f5 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -31,6 +31,7 @@ on: permissions: contents: read + id-token: write env: DD_ENV: "ci" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index db19d27ffd582..8ca10e1127a0d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,6 +16,9 @@ env: jobs: coverage: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -31,9 +34,10 @@ jobs: - name: "Generate code coverage" run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info + - uses: ./.github/actions/dd-token + - name: "Upload coverage to Datadog" env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci run: datadog-ci coverage upload lcov.info diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 74b782aa7d336..641f9ceb96d4b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -23,7 +23,6 @@ env: TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true @@ -34,6 +33,9 @@ jobs: test-integration: runs-on: ubuntu-24.04 timeout-minutes: 40 + permissions: + contents: read + id-token: write if: inputs.if || github.event_name == 'workflow_dispatch' steps: - name: (PR comment) Get PR branch @@ -51,6 +53,8 @@ jobs: if: ${{ github.event_name != 'issue_comment' }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/dd-token + - run: bash scripts/environment/prepare.sh --modules=rustup,datadog-ci - run: make test-integration-${{ inputs.test_name }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2c1e91c57a8e7..833a7ae13d5de 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,7 +23,6 @@ permissions: env: CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} @@ -65,6 +64,9 @@ jobs: integration-tests: runs-on: ubuntu-24.04-8core + permissions: + contents: read + id-token: write needs: - changes - build-test-runner @@ -141,6 +143,9 @@ jobs: with: submodules: "recursive" + - uses: ./.github/actions/dd-token + if: steps.check.outputs.should_run == 'true' + - uses: ./.github/actions/setup if: steps.check.outputs.should_run == 'true' with: @@ -167,6 +172,9 @@ jobs: e2e-tests: runs-on: ubuntu-24.04-8core + permissions: + contents: read + id-token: write needs: - changes - build-test-runner @@ -205,6 +213,9 @@ jobs: with: submodules: "recursive" + - uses: ./.github/actions/dd-token + if: steps.check.outputs.should_run == 'true' + - uses: ./.github/actions/setup if: steps.check.outputs.should_run == 'true' with: diff --git a/.github/workflows/master_merge_queue.yml b/.github/workflows/master_merge_queue.yml index 8f6866f42dfe2..3eaffecfdb2cd 100644 --- a/.github/workflows/master_merge_queue.yml +++ b/.github/workflows/master_merge_queue.yml @@ -21,6 +21,7 @@ on: permissions: contents: read + id-token: write concurrency: # `github.ref` is unique for MQ runs and PRs @@ -30,7 +31,6 @@ concurrency: env: CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 9411f2844e264..f74c6681d8ea5 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -13,14 +13,18 @@ permissions: jobs: static-analysis: runs-on: ubuntu-latest - env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} - DD_APP_KEY: ${{ secrets.DD_APP_KEY }} + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: dd-token + uses: ./.github/actions/dd-token + with: + policy: public-vectordotdev-vector-static-analysis + - name: Datadog Static Analyzer - if: ${{ env.DD_API_KEY != '' }} uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 with: dd_api_key: ${{ env.DD_API_KEY }} diff --git a/.github/workflows/test-make-command.yml b/.github/workflows/test-make-command.yml index 79c27d54792f4..2a93f9016fcd1 100644 --- a/.github/workflows/test-make-command.yml +++ b/.github/workflows/test-make-command.yml @@ -32,17 +32,22 @@ permissions: jobs: run-make-command: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 90 env: CARGO_INCREMENTAL: 0 DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} steps: - name: Checkout branch uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref }} + - uses: ./.github/actions/dd-token + if: ${{ inputs.upload_test_results }} + - uses: ./.github/actions/setup with: rust: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84a4c31226b1f..7e5c28aec476c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,6 @@ concurrency: env: CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} VECTOR_LOG: vector=debug VERBOSE: true CI: true @@ -57,6 +56,9 @@ jobs: test: needs: changes + permissions: + contents: read + id-token: write if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} uses: ./.github/workflows/unit-tests.yml with: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e3b6501247858..63c9de7d19351 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -50,17 +50,21 @@ env: CARGO_INCREMENTAL: "0" CI: true DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} jobs: tests: name: Tests runs-on: ubuntu-24.04-8core + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref }} + - uses: ./.github/actions/dd-token + - uses: ./.github/actions/setup with: rust: true From 7815f26ef4f357c7e3e90048e4287728c9506423 Mon Sep 17 00:00:00 2001 From: "shalk(xiao kun)" Date: Sat, 25 Apr 2026 02:43:56 +0800 Subject: [PATCH 110/364] chore(website): add Docker support for local development (#25237) * feat(website): add Docker support for local development Adds Dockerfile, docker-compose.yml, and README instructions so the Vector website can be previewed locally without installing Hugo, CUE, or Node.js. The image detects host architecture via uname -m to support both amd64 and arm64. VRL function docs still require a one-time `make generate-vrl-docs` on the host (needs Rust/vdev). Co-Authored-By: Claude Sonnet 4.6 * Update website/Dockerfile Co-authored-by: Thomas * docs(website): note Docker setup is experimental and not CI-enforced Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Thomas --- website/Dockerfile | 44 ++++++++++++++++++++++++++++++++++++++ website/README.md | 24 +++++++++++++++++++++ website/docker-compose.yml | 15 +++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 website/Dockerfile create mode 100644 website/docker-compose.yml diff --git a/website/Dockerfile b/website/Dockerfile new file mode 100644 index 0000000000000..e9946941bb459 --- /dev/null +++ b/website/Dockerfile @@ -0,0 +1,44 @@ +FROM debian:bookworm-slim + +ARG HUGO_VERSION=0.154.5 +ARG CUE_VERSION=0.10.0 +ARG NODE_VERSION=24 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +# Hugo extended (required for Sass/PostCSS processing) +# uname -m: x86_64 -> amd64, aarch64 -> arm64 +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') && \ + curl -fsSL https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-${ARCH}.tar.gz \ + | tar -xz -C /usr/local/bin hugo + +# CUE CLI +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') && \ + curl -fsSL https://github.com/cue-lang/cue/releases/download/v${CUE_VERSION}/cue_v${CUE_VERSION}_linux_${ARCH}.tar.gz \ + | tar -xz -C /usr/local/bin cue + +# Node.js + Yarn +RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g yarn \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app/website + +EXPOSE 1313 + +# generate-vrl-docs (requires Rust/vdev) must be run on the host first. +# Call cue directly instead of scripts/cue.sh to avoid its `git rev-parse` dependency. +ENTRYPOINT ["/bin/sh", "-c", \ + "if [ -z \"$(find cue/reference/remap/functions -name '*.cue' 2>/dev/null | head -1)\" ]; then \ + echo 'ERROR: cue/reference/remap/functions/ is empty.'; \ + echo 'Run on the host first: make generate-vrl-docs'; \ + exit 1; \ + fi && \ + yarn && \ + cp -f /app/Cargo.lock data/cargo-lock.toml && \ + rm -f data/docs.json && find cue -name '*.cue' | xargs cue export --all-errors --outfile data/docs.json && \ + yarn config-examples && \ + hugo server --bind 0.0.0.0 --port 1313 --buildDrafts --buildFuture --environment development"] diff --git a/website/README.md b/website/README.md index 47475945dbf9d..79a565b89fb7b 100644 --- a/website/README.md +++ b/website/README.md @@ -136,6 +136,30 @@ This builds all the necessary [prereqs](#prerequisites) for the site and starts When you make changes to the Markdown sources, Sass/CSS, or JavaScript, the site re-builds and Hugo automatically reloads the page that you're on. If you make changes to the [structured data](#structured-data) sources, however, you need to stop the server and run `make serve` again. +### Run the site with Docker + +If you don't want to install Hugo, CUE, or Node.js locally, you can use Docker instead. You still need Rust and [vdev] on the host for one step. + +> **Note:** This Docker setup is experimental and not currently enforced by CI. It may break as dependencies or the build process evolve. + +**Step 1:** Generate VRL function docs (required once; only re-run when VRL function signatures change): + +```shell +make generate-vrl-docs +``` + +This generates CUE files into `cue/reference/remap/functions/` from the Rust source. These files are gitignored and must exist before the site can build. + +**Step 2:** Build and start the site: + +```shell +cd website +docker compose build +docker compose up +``` + +Navigate to http://localhost:1313. The site source is mounted as a volume so Hugo's live-reload works for Markdown, CSS, and JavaScript changes. If you modify [structured data](#structured-data) sources, restart the container. + ### Add a new version of Vector 1. Add the new version to the `versions` list in [`cue/reference/versions.cue`](./cue/reference/versions.cue). Make sure to preserve reverse ordering. diff --git a/website/docker-compose.yml b/website/docker-compose.yml new file mode 100644 index 0000000000000..2f58c9a12c815 --- /dev/null +++ b/website/docker-compose.yml @@ -0,0 +1,15 @@ +services: + website: + build: . + ports: + - "1313:1313" + volumes: + - .:/app/website + - ../Cargo.lock:/app/Cargo.lock:ro + - ../scripts:/app/scripts:ro + - website_node_modules:/app/website/node_modules + environment: + - CI=false + +volumes: + website_node_modules: From 74574a360112a4bb212915b896471320fb157b96 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Apr 2026 14:50:53 -0400 Subject: [PATCH 111/364] chore(vdev): add linux arm64 to publish matrix and bump to 0.3.2 (#25260) --- .github/workflows/vdev_publish.yml | 2 ++ Cargo.lock | 2 +- scripts/environment/prepare.sh | 2 +- vdev/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml index 3a46822098beb..80200d395f6ca 100644 --- a/.github/workflows/vdev_publish.yml +++ b/.github/workflows/vdev_publish.yml @@ -18,6 +18,8 @@ jobs: target: aarch64-apple-darwin - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu + - os: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu steps: - name: Checkout Vector diff --git a/Cargo.lock b/Cargo.lock index 57f9a916e8812..16d4bb553fbce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12241,7 +12241,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.1" +version = "0.3.2" dependencies = [ "anyhow", "cfg-if", diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 02e100999c8de..b5c63c02f9d69 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -39,7 +39,7 @@ CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" # npm tool versions are defined in scripts/environment/npm-tools/package.json # and pinned (including transitive deps) in npm-tools/package-lock.json. -VDEV_VERSION="0.3.0" +VDEV_VERSION="0.3.2" ALL_MODULES=( rustup diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 2a0f6169a160e..ff0265fddfc3e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.1" +version = "0.3.2" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0" From 3b043d33b95670ef1cbd2f5af274fa25ebb1db84 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Apr 2026 14:55:51 -0400 Subject: [PATCH 112/364] feat(greptimedb sink)!: add support for GreptimeDB v1.0.0 (#25178) * feat(greptimedb sink)!: upgrade greptimedb-ingester to published crate v0.16.0 * test(greptimedb): add latest to integration test version matrix * fix(greptimedb sink): reject unsupported gRPC compression types * refactor(greptimedb sink): make grpc_compression a typed enum * Format * Update licenses * Update generated docs * fix(greptimedb sink): fail early when TLS paths are missing * fix(greptimedb sink): handle partial TLS configs gracefully and document gzip removal * Update greptimedb-ingester and add back gzip support * Reject incomplete TLS configs rather than falling back to non-TLS * Match on all TlsConfig entries to emit warning log * Update generated docs --- Cargo.lock | 700 +++++++++++++----- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 4 + .../greptimedb_ingester_crate.breaking.md | 3 + src/sinks/greptimedb/metrics/config.rs | 8 +- src/sinks/greptimedb/metrics/request.rs | 3 +- .../greptimedb/metrics/request_builder.rs | 6 +- src/sinks/greptimedb/metrics/service.rs | 116 +-- src/sinks/greptimedb/mod.rs | 14 + tests/integration/greptimedb/config/test.yaml | 4 +- .../components/sinks/generated/greptimedb.cue | 18 +- .../sinks/generated/greptimedb_metrics.cue | 18 +- 12 files changed, 623 insertions(+), 273 deletions(-) create mode 100644 changelog.d/greptimedb_ingester_crate.breaking.md diff --git a/Cargo.lock b/Cargo.lock index 16d4bb553fbce..0ad2d1635d855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,18 +343,39 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", +] + +[[package]] +name = "arrow" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +dependencies = [ + "arrow-arith 58.1.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-csv", + "arrow-data 58.1.0", + "arrow-ipc 58.1.0", + "arrow-json 58.1.0", + "arrow-ord 58.1.0", + "arrow-row 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "arrow-string 58.1.0", ] [[package]] @@ -363,14 +384,28 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] +[[package]] +name = "arrow-arith" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "56.2.0" @@ -378,9 +413,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash 0.8.11", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "chrono-tz", "half", @@ -388,6 +423,25 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-array" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.0", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "56.2.0" @@ -399,17 +453,29 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "atoi", "base64 0.22.1", "chrono", @@ -419,30 +485,116 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-ord 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +dependencies = [ + "arrow-array 58.1.0", + "arrow-cast 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] +[[package]] +name = "arrow-data" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +dependencies = [ + "arrow-buffer 58.1.0", + "arrow-schema 58.1.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302b2e036335f3f04d65dad3f74ff1f2aae6dc671d6aa04dc6b61193761e16fb" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", + "base64 0.22.1", + "bytes", + "futures", + "prost 0.14.3", + "prost-types 0.14.3", + "tonic 0.14.5", + "tonic-prost", +] + [[package]] name = "arrow-ipc" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "flatbuffers", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers 25.9.23", +] + +[[package]] +name = "arrow-ipc" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "flatbuffers 25.9.23", + "lz4_flex 0.13.0", + "zstd", ] [[package]] @@ -451,11 +603,11 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", "indexmap 2.12.0", @@ -467,17 +619,54 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "half", + "indexmap 2.12.0", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", +] + +[[package]] +name = "arrow-ord" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", ] [[package]] @@ -486,10 +675,23 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "half", ] @@ -499,6 +701,16 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +[[package]] +name = "arrow-schema" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "arrow-select" version = "56.2.0" @@ -506,30 +718,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] +[[package]] +name = "arrow-select" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +dependencies = [ + "ahash 0.8.11", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "num-traits", +] + [[package]] name = "arrow-string" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "memchr", "num", "regex", "regex-syntax", ] +[[package]] +name = "arrow-string" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "ascii-canvas" version = "4.0.0" @@ -1548,7 +1791,7 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.32", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -1576,19 +1819,44 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", "pin-project-lite", "rustversion", "serde", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower 0.4.13", "tower-layer", "tower-service", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper 1.0.2", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -1621,7 +1889,25 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] @@ -1950,7 +2236,7 @@ checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ "darling 0.21.3", "ident_case", - "prettyplease 0.2.37", + "prettyplease", "proc-macro2 1.0.106", "quote 1.0.44", "rustversion", @@ -2425,7 +2711,7 @@ name = "codecs" version = "0.1.0" dependencies = [ "apache-avro 0.20.0", - "arrow", + "arrow 56.2.0", "async-trait", "bytes", "chrono", @@ -2515,6 +2801,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width 0.2.0", +] + [[package]] name = "community-id" version = "0.2.3" @@ -2553,8 +2849,8 @@ dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.13.2", - "zstd-safe 7.2.1", + "zstd", + "zstd-safe", ] [[package]] @@ -4008,6 +4304,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flatbuffers" version = "25.9.23" @@ -4408,37 +4714,54 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.9.0#396206c2801b5a3ec51bfe8984c66b686da910e6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25fc58f8ba94c488192d5928f24caf16c503fb83f4784f617e83febe08647b08" dependencies = [ - "prost 0.12.6", + "prost 0.14.3", + "prost-types 0.14.3", "serde", "serde_json", "strum 0.25.0", "strum_macros 0.25.3", - "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic 0.14.5", + "tonic-prost", ] [[package]] name = "greptimedb-ingester" -version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=f7243393808640f5123b0d5b7b798da591a4df6e#f7243393808640f5123b0d5b7b798da591a4df6e" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ee8f8f967c2b8183136920bdb0a36d06618cc9402f901495c0da66cb2cc74b" dependencies = [ + "arrow 58.1.0", + "arrow-array 58.1.0", + "arrow-flight", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", + "async-stream", + "async-trait", + "base64 0.22.1", "dashmap", "derive_builder", "enum_dispatch", + "flatbuffers 24.12.23", "futures", "futures-util", "greptime-proto", + "hyper 1.7.0", + "lazy_static", "parking_lot", - "prost 0.12.6", + "prost 0.14.3", "rand 0.9.4", + "serde", + "serde_json", "snafu 0.8.9", "tokio", "tokio-stream", - "tonic 0.11.0", - "tonic-build 0.9.2", - "tower 0.4.13", + "tokio-util", + "toml", + "tonic 0.14.5", + "tower 0.5.3", ] [[package]] @@ -6309,7 +6632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "823ec7bedb1819b11633bd583ae981b0082db08492b0c3396412b85dd329ffee" dependencies = [ "cc", - "which 5.0.0", + "which", ] [[package]] @@ -6340,6 +6663,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + [[package]] name = "macaddr" version = "1.0.1" @@ -6433,6 +6765,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "matrixmultiply" version = "0.3.8" @@ -7065,9 +7403,9 @@ checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -7437,7 +7775,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic-build", "vector-core", "vector-lookup", "vrl", @@ -7572,20 +7910,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "base64 0.22.1", "bytes", "chrono", "flate2", "half", "hashbrown 0.16.0", - "lz4_flex", + "lz4_flex 0.11.6", "num", "num-bigint", "paste", @@ -7593,7 +7931,7 @@ dependencies = [ "snap", "thrift", "twox-hash", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -7996,16 +8334,6 @@ dependencies = [ "pad", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2 1.0.106", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -8173,16 +8501,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", -] - [[package]] name = "prost" version = "0.12.6" @@ -8204,25 +8522,13 @@ dependencies = [ ] [[package]] -name = "prost-build" -version = "0.11.9" +name = "prost" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.10.5", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease 0.1.25", - "prost 0.11.9", - "prost-types 0.11.9", - "regex", - "syn 1.0.109", - "tempfile", - "which 4.4.2", + "prost-derive 0.14.3", ] [[package]] @@ -8238,7 +8544,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.37", + "prettyplease", "prost 0.12.6", "prost-types 0.12.6", "regex", @@ -8258,7 +8564,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.37", + "prettyplease", "prost 0.13.5", "prost-types 0.13.5", "regex", @@ -8268,25 +8574,25 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2 1.0.106", "quote 1.0.44", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -8294,9 +8600,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", @@ -8319,15 +8625,6 @@ dependencies = [ "serde-value", ] -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", -] - [[package]] name = "prost-types" version = "0.12.6" @@ -8346,6 +8643,15 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "psl" version = "2.1.22" @@ -8426,7 +8732,7 @@ dependencies = [ "tokio-util", "url", "uuid", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -9129,7 +9435,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", "tokio-rustls 0.26.2", @@ -10790,9 +11096,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -11480,7 +11786,6 @@ dependencies = [ "tower-layer", "tower-service", "tracing 0.1.44", - "zstd 0.12.4", ] [[package]] @@ -11514,16 +11819,35 @@ dependencies = [ ] [[package]] -name = "tonic-build" -version = "0.9.2" +name = "tonic" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ - "prettyplease 0.1.25", - "proc-macro2 1.0.106", - "prost-build 0.11.9", - "quote 1.0.44", - "syn 1.0.109", + "async-trait", + "axum 0.8.9", + "base64 0.22.1", + "bytes", + "flate2", + "h2 0.4.13", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-timeout 0.5.1", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2 0.6.0", + "sync_wrapper 1.0.2", + "tokio", + "tokio-rustls 0.26.2", + "tokio-stream", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing 0.1.44", + "zstd", ] [[package]] @@ -11532,7 +11856,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ - "prettyplease 0.2.37", + "prettyplease", "proc-macro2 1.0.106", "prost-build 0.12.6", "quote 1.0.44", @@ -11552,6 +11876,17 @@ dependencies = [ "tonic 0.11.0", ] +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost 0.14.3", + "tonic 0.14.5", +] + [[package]] name = "tonic-reflection" version = "0.11.0" @@ -11596,7 +11931,7 @@ dependencies = [ "indexmap 2.12.0", "pin-project-lite", "slab", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-util", "tower-layer", @@ -12281,8 +12616,8 @@ dependencies = [ "approx", "arc-swap", "arr_macro", - "arrow", - "arrow-schema", + "arrow 56.2.0", + "arrow-schema 56.2.0", "assert_cmd", "async-compression", "async-nats", @@ -12455,7 +12790,7 @@ dependencies = [ "tokio-util", "toml", "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic-build", "tonic-health", "tonic-reflection", "tower 0.5.3", @@ -12482,7 +12817,7 @@ dependencies = [ "windows 0.58.0", "windows-service", "wiremock", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -12498,7 +12833,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic-build", "tonic-health", ] @@ -12962,7 +13297,7 @@ dependencies = [ "jsonschema", "lalrpop", "lalrpop-util", - "lz4_flex", + "lz4_flex 0.11.6", "md-5", "mlua", "nom 8.0.0", @@ -13023,7 +13358,7 @@ dependencies = [ "webbrowser", "woothee", "xxhash-rust", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -13299,18 +13634,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.40", -] - [[package]] name = "which" version = "5.0.0" @@ -13981,7 +14304,7 @@ dependencies = [ "anyhow", "heck 0.5.0", "indexmap 2.12.0", - "prettyplease 0.2.37", + "prettyplease", "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", @@ -13995,7 +14318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ "anyhow", - "prettyplease 0.2.37", + "prettyplease", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -14202,32 +14525,13 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" -[[package]] -name = "zstd" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" -dependencies = [ - "zstd-safe 6.0.6", -] - [[package]] name = "zstd" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.2.1", -] - -[[package]] -name = "zstd-safe" -version = "6.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1045e07aae1a1..70f9fc12a8acc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -345,7 +345,7 @@ deadpool = { version = "0.13.0", default-features = false, features = ["managed" hex = { version = "0.4.3", default-features = false, optional = true } # GreptimeDB -greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "f7243393808640f5123b0d5b7b798da591a4df6e", optional = true } +greptimedb-ingester = { version = "0.17.0", default-features = false, optional = true } # External libs arc-swap = { workspace = true, default-features = false, optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index eb9e179e57c2b..ea2ff8e85d12e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -32,7 +32,9 @@ arrow-arith,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-buffer,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-cast,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-csv,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-data,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-flight,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-ipc,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-json,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-ord,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow @@ -165,6 +167,7 @@ codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zab colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham combine,https://github.com/Marwes/combine,MIT,Markus Westerlind +comfy-table,https://github.com/nukesor/comfy-table,MIT,Arne Beer community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " @@ -789,6 +792,7 @@ toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Auth toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-health,https://github.com/hyperium/tonic,MIT,James Nugent +tonic-prost,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-reflection,https://github.com/hyperium/tonic,MIT,"James Nugent , Samani G. Gikandi " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers diff --git a/changelog.d/greptimedb_ingester_crate.breaking.md b/changelog.d/greptimedb_ingester_crate.breaking.md new file mode 100644 index 0000000000000..79984693ef8e1 --- /dev/null +++ b/changelog.d/greptimedb_ingester_crate.breaking.md @@ -0,0 +1,3 @@ +The `greptimedb_metrics` and `greptimedb_logs` sinks now require GreptimeDB v1.x. Users running GreptimeDB v0.x must upgrade their GreptimeDB instance before upgrading Vector. + +authors: thomasqueirozb diff --git a/src/sinks/greptimedb/metrics/config.rs b/src/sinks/greptimedb/metrics/config.rs index 4261486a05706..8f91392325698 100644 --- a/src/sinks/greptimedb/metrics/config.rs +++ b/src/sinks/greptimedb/metrics/config.rs @@ -2,7 +2,7 @@ use vector_lib::{configurable::configurable_component, sensitive_string::Sensiti use crate::sinks::{ greptimedb::{ - GreptimeDBDefaultBatchSettings, default_dbname, + GreptimeDBDefaultBatchSettings, GrpcCompression, default_dbname, metrics::{ request::GreptimeDBGrpcRetryLogic, request_builder::RequestBuilderOptions, @@ -85,11 +85,9 @@ pub struct GreptimeDBMetricsConfig { #[configurable(metadata(docs::examples = "password"))] #[serde(default)] pub password: Option, - /// Set gRPC compression encoding for the request - /// Default to none, `gzip` or `zstd` is supported. - #[configurable(metadata(docs::examples = "gzip"))] + /// Set gRPC compression encoding for the request. #[serde(default)] - pub grpc_compression: Option, + pub grpc_compression: GrpcCompression, #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/greptimedb/metrics/request.rs b/src/sinks/greptimedb/metrics/request.rs index eedfc6f26f2d8..68767844412dc 100644 --- a/src/sinks/greptimedb/metrics/request.rs +++ b/src/sinks/greptimedb/metrics/request.rs @@ -1,6 +1,7 @@ use std::num::NonZeroUsize; -use greptimedb_ingester::{Error as GreptimeError, api::v1::*}; +use greptimedb_ingester::Error as GreptimeError; +use greptimedb_ingester::api::v1::RowInsertRequests; use vector_lib::event::Metric; use crate::sinks::{ diff --git a/src/sinks/greptimedb/metrics/request_builder.rs b/src/sinks/greptimedb/metrics/request_builder.rs index 1980d67e6a78c..e596ae06e0cee 100644 --- a/src/sinks/greptimedb/metrics/request_builder.rs +++ b/src/sinks/greptimedb/metrics/request_builder.rs @@ -1,5 +1,8 @@ use chrono::Utc; -use greptimedb_ingester::{api::v1::*, helpers::values::*}; +use greptimedb_ingester::api::v1::{ + ColumnDataType, ColumnSchema, Row, RowInsertRequest, Rows, SemanticType, Value, +}; +use greptimedb_ingester::helpers::values::{f64_value, string_value, timestamp_millisecond_value}; use vector_lib::{ event::{ Metric, MetricValue, @@ -238,6 +241,7 @@ fn tag_column(name: &str) -> ColumnSchema { #[cfg(test)] mod tests { + use greptimedb_ingester::api::v1::value; use similar_asserts::assert_eq; use super::*; diff --git a/src/sinks/greptimedb/metrics/service.rs b/src/sinks/greptimedb/metrics/service.rs index 77ac61593302a..e3bdc4a07b1b6 100644 --- a/src/sinks/greptimedb/metrics/service.rs +++ b/src/sinks/greptimedb/metrics/service.rs @@ -1,16 +1,22 @@ use std::{sync::Arc, task::Poll}; use greptimedb_ingester::{ - Client, ClientBuilder, Compression, Database, Error as GreptimeError, - api::v1::{auth_header::AuthScheme, *}, - channel_manager::*, + Error as GreptimeError, GrpcCompression as IngesterGrpcCompression, + api::v1::Basic, + api::v1::auth_header::AuthScheme, + channel_manager::{ChannelConfig, ChannelManager, ClientTlsOption}, + client::Client, + database::Database, }; use vector_lib::sensitive_string::SensitiveString; use crate::sinks::{ - greptimedb::metrics::{ - config::GreptimeDBMetricsConfig, - request::{GreptimeDBGrpcBatchOutput, GreptimeDBGrpcRequest}, + greptimedb::{ + GrpcCompression, + metrics::{ + config::GreptimeDBMetricsConfig, + request::{GreptimeDBGrpcBatchOutput, GreptimeDBGrpcRequest}, + }, }, prelude::*, }; @@ -22,49 +28,63 @@ pub struct GreptimeDBGrpcService { } fn new_client_from_config(config: &GreptimeDBGrpcServiceConfig) -> crate::Result { - let mut builder = ClientBuilder::default().peers(vec![&config.endpoint]); + let send_compression_encoding = match config.compression { + GrpcCompression::None => None, + GrpcCompression::Gzip => Some(IngesterGrpcCompression::Gzip), + GrpcCompression::Zstd => Some(IngesterGrpcCompression::Zstd), + }; + + let mut channel_config = ChannelConfig { + send_compression_encoding, + ..Default::default() + }; + + let channel_manager = if let Some(tls_config) = &config.tls { + let TlsConfig { + verify_certificate, + verify_hostname, + alpn_protocols, + ca_file, + crt_file, + key_file, + key_pass, + server_name, + } = tls_config; + + if verify_certificate.is_some() + || verify_hostname.is_some() + || alpn_protocols.is_some() + || key_pass.is_some() + || server_name.is_some() + { + warn!( + message = "TlsConfig: verify_certificate, verify_hostname, alpn_protocols, key_pass and server_name are not supported by greptimedb client at the moment." + ); + } - if let Some(compression) = config.compression.as_ref() { - let compression = match compression.as_str() { - "gzip" => Compression::Gzip, - "zstd" => Compression::Zstd, + // The greptimedb ingester requires all three TLS paths (ca_file, crt_file, + // key_file) to be set. Refuse a partial config rather than downgrading to plaintext. + match (ca_file, crt_file, key_file) { + (Some(ca), Some(crt), Some(key)) => { + channel_config.client_tls = Some(ClientTlsOption { + server_ca_cert_path: ca.to_string_lossy().into_owned(), + client_cert_path: crt.to_string_lossy().into_owned(), + client_key_path: key.to_string_lossy().into_owned(), + }); + ChannelManager::with_tls_config(channel_config).map_err(Box::new)? + } _ => { - warn!(message = "Unknown gRPC compression type: {compression}, disabled."); - Compression::None + return Err( + "GreptimeDB TLS requires ca_file, crt_file, and key_file to all be set.".into(), + ); } - }; - builder = builder.compression(compression); - } - - if let Some(tls_config) = &config.tls { - let channel_config = ChannelConfig { - client_tls: Some(try_from_tls_config(tls_config)?), - ..Default::default() - }; - - builder = builder - .channel_manager(ChannelManager::with_tls_config(channel_config).map_err(Box::new)?); - } - - Ok(builder.build()) -} - -fn try_from_tls_config(tls_config: &TlsConfig) -> crate::Result { - if tls_config.key_pass.is_some() - || tls_config.alpn_protocols.is_some() - || tls_config.verify_certificate.is_some() - || tls_config.verify_hostname.is_some() - { - warn!( - message = "TlsConfig: key_pass, alpn_protocols, verify_certificate and verify_hostname are not supported by greptimedb client at the moment." - ); - } + } + } else { + ChannelManager::with_config(channel_config) + }; + let client = Client::with_manager_and_urls(channel_manager, vec![&config.endpoint]); - Ok(ClientTlsOption { - server_ca_cert_path: tls_config.ca_file.clone(), - client_cert_path: tls_config.crt_file.clone(), - client_key_path: tls_config.key_file.clone(), - }) + Ok(client) } impl GreptimeDBGrpcService { @@ -103,7 +123,7 @@ impl Service for GreptimeDBGrpcService { Box::pin(async move { let metadata = req.metadata; - let result = client.row_insert(req.items).await?; + let result = client.insert(req.items).await?; Ok(GreptimeDBGrpcBatchOutput { _item_count: result, @@ -119,7 +139,7 @@ pub(super) struct GreptimeDBGrpcServiceConfig { dbname: String, username: Option, password: Option, - compression: Option, + compression: GrpcCompression, tls: Option, } @@ -130,7 +150,7 @@ impl From<&GreptimeDBMetricsConfig> for GreptimeDBGrpcServiceConfig { dbname: val.dbname.clone(), username: val.username.clone(), password: val.password.clone(), - compression: val.grpc_compression.clone(), + compression: val.grpc_compression, tls: val.tls.clone(), } } diff --git a/src/sinks/greptimedb/mod.rs b/src/sinks/greptimedb/mod.rs index 8116681652dc2..5369992e21c46 100644 --- a/src/sinks/greptimedb/mod.rs +++ b/src/sinks/greptimedb/mod.rs @@ -4,6 +4,20 @@ use crate::sinks::prelude::*; mod logs; mod metrics; +/// Compression algorithm for gRPC requests to GreptimeDB. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default)] +#[serde(rename_all = "lowercase")] +enum GrpcCompression { + /// No compression. + #[default] + None, + /// Gzip compression. + Gzip, + /// Zstandard compression. + Zstd, +} + fn default_dbname() -> String { greptimedb_ingester::DEFAULT_SCHEMA_NAME.to_string() } diff --git a/tests/integration/greptimedb/config/test.yaml b/tests/integration/greptimedb/config/test.yaml index 2ec3984938617..8a79569ac06fa 100644 --- a/tests/integration/greptimedb/config/test.yaml +++ b/tests/integration/greptimedb/config/test.yaml @@ -10,9 +10,7 @@ runner: no_proxy: greptimedb matrix: - # Temporarily pegging to the latest known stable release - # since using `latest` is failing consistently. - version: [v0.13.2] + version: [v1.0.0, latest] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch diff --git a/website/cue/reference/components/sinks/generated/greptimedb.cue b/website/cue/reference/components/sinks/generated/greptimedb.cue index 90cbf9c459d62..27abfd61e2960 100644 --- a/website/cue/reference/components/sinks/generated/greptimedb.cue +++ b/website/cue/reference/components/sinks/generated/greptimedb.cue @@ -92,14 +92,16 @@ generated: components: sinks: greptimedb: configuration: { type: string: examples: ["example.com:4001"] } grpc_compression: { - description: """ - Set gRPC compression encoding for the request - Default to none, `gzip` or `zstd` is supported. - """ - required: false - type: string: examples: [ - "gzip", - ] + description: "Set gRPC compression encoding for the request." + required: false + type: string: { + default: "none" + enum: { + gzip: "Gzip compression." + none: "No compression." + zstd: "Zstandard compression." + } + } } new_naming: { description: """ diff --git a/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue b/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue index aea1d76a0eb46..eb262d81fbea5 100644 --- a/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue +++ b/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue @@ -92,14 +92,16 @@ generated: components: sinks: greptimedb_metrics: configuration: { type: string: examples: ["example.com:4001"] } grpc_compression: { - description: """ - Set gRPC compression encoding for the request - Default to none, `gzip` or `zstd` is supported. - """ - required: false - type: string: examples: [ - "gzip", - ] + description: "Set gRPC compression encoding for the request." + required: false + type: string: { + default: "none" + enum: { + gzip: "Gzip compression." + none: "No compression." + zstd: "Zstandard compression." + } + } } new_naming: { description: """ From 61cc4d84d2966e7552a308d5df5a0f03305d7eed Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 09:40:18 -0400 Subject: [PATCH 113/364] fix(ci): make publish-s3 wait for generate-sha256sum (#25265) --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d5093f229017c..5ccb5ea82c968 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -382,6 +382,7 @@ jobs: - deb-verify - rpm-verify - macos-verify + - generate-sha256sum env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} From 454b096abcfd609b4e317b83fa4ec1ec79af5da8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 27 Apr 2026 10:14:28 -0400 Subject: [PATCH 114/364] docs(deprecations): add datadog_metrics series v1 deprecation entry (#25271) Co-authored-by: Claude Opus 4.7 (1M context) --- docs/DEPRECATIONS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index b57db75f736af..9c36ff3eb7859 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -17,6 +17,7 @@ For example: - `v0.50.0` | `http-server-encoding` | The `encoding` field will be removed. Use `decoding` and `framing` instead. - `v0.53.0` | `buffer-bytes-events-metrics` | The `buffer_byte_size` and `buffer_events` gauges are deprecated in favor of the `buffer_size_bytes`/`buffer_size_events` metrics described in `docs/specs/buffer.md`. - `v0.58.0` | `azure-monitor-logs-sink` | The `azure_monitor_logs` sink is deprecated in favor of the new `azure_logs_ingestion` sink, which uses the Azure Monitor Logs Ingestion API. Users should migrate before Microsoft ends support for the old Data Collector API (scheduled for September 2026). +- `v0.58.0` | `datadog-metrics-series-v1` | The `series_api_version: v1` option on the `datadog_metrics` sink is deprecated in favor of `v2` (the default). The v1 series endpoint (`/api/v1/series`) is a legacy endpoint; users should remove `series_api_version: v1` from their configuration or set it to `v2`. ## To be migrated From aa9265d6026f7d5998cada6a50e9c66790fbf054 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 10:47:04 -0400 Subject: [PATCH 115/364] chore(deps): add dependabot config for scripts/environment/npm-tools (#25175) * chore(deps): add dependabot config for scripts/environment/npm-tools * Set open-pull-requests-limit: 0 --- .github/dependabot.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 609da972baa98..ade867f95145f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -176,3 +176,19 @@ updates: npm_and_yarn: patterns: - "*" + - package-ecosystem: "npm" + directory: "/scripts/environment/npm-tools" + schedule: + interval: "monthly" + time: "05:00" # UTC + labels: + - "domain: deps" + - "no-changelog" + commit-message: + prefix: "chore(deps)" + # Disable version updates; security updates bypass this limit + open-pull-requests-limit: 0 + groups: + npm_and_yarn: + patterns: + - "*" From 69fd3b6f7e544a3f4bbba1d6d7e882a67387b081 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:56:51 -0400 Subject: [PATCH 116/364] chore(ci): bump github/codeql-action from 4.32.4 to 4.35.2 (#25280) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.32.4 to 4.35.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/89a39a4e59826350b863aa6b6252a07ad50cf83e...95e58e9a2cdfd71adc6e0353d5c52f41a045d225) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.35.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7a8fd8c9a7d47..391406f824e27 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4 + uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 with: sarif_file: results.sarif From 413caac10f64475b509687000317e06e55de55e0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 11:18:47 -0400 Subject: [PATCH 117/364] fix(ci): allow dd-token federation to fail on fork PRs (#25284) --- .github/actions/dd-token/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/dd-token/action.yml b/.github/actions/dd-token/action.yml index 8ea92dc1c2dd5..f2b6b32eb0a16 100644 --- a/.github/actions/dd-token/action.yml +++ b/.github/actions/dd-token/action.yml @@ -10,8 +10,11 @@ inputs: runs: using: "composite" steps: + # Fork PRs don't get `id-token: write`, so federation cannot succeed. + # Allow the step to fail and let downstream steps run without DD credentials. - name: Federate Datadog token id: dd-sts + continue-on-error: true uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 with: policy: ${{ inputs.policy }} @@ -22,7 +25,9 @@ runs: DD_STS_API_KEY: ${{ steps.dd-sts.outputs.api_key }} DD_STS_APP_KEY: ${{ steps.dd-sts.outputs.app_key }} run: | - echo "DD_API_KEY=${DD_STS_API_KEY}" >> "$GITHUB_ENV" + if [ -n "${DD_STS_API_KEY}" ]; then + echo "DD_API_KEY=${DD_STS_API_KEY}" >> "$GITHUB_ENV" + fi if [ -n "${DD_STS_APP_KEY}" ]; then echo "DD_APP_KEY=${DD_STS_APP_KEY}" >> "$GITHUB_ENV" fi From ba84da5d8965aef1d5eb5455dc9f4d723177e914 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:01:24 -0400 Subject: [PATCH 118/364] chore(ci): bump actions/github-script from 7 to 9 (#25278) * chore(ci): bump actions/github-script from 7 to 9 Bumps [actions/github-script](https://github.com/actions/github-script) from 7 to 9. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v7...v9) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: '9' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Use pinned github-script sha --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- .github/workflows/add_wip_label.yml | 2 +- .github/workflows/check_generated_vrl_docs_comment.yml | 2 +- .github/workflows/create_preview_sites.yml | 10 +++++----- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/preview_site_trigger.yml | 2 +- .github/workflows/remove_wip_label.yml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_wip_label.yml index 9a1260b693e81..7924d4fade926 100644 --- a/.github/workflows/add_wip_label.yml +++ b/.github/workflows/add_wip_label.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Check member review state id: check - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const { data: reviews } = await github.rest.pulls.listReviews({ diff --git a/.github/workflows/check_generated_vrl_docs_comment.yml b/.github/workflows/check_generated_vrl_docs_comment.yml index e455203f75a6b..faa429af4de9b 100644 --- a/.github/workflows/check_generated_vrl_docs_comment.yml +++ b/.github/workflows/check_generated_vrl_docs_comment.yml @@ -26,7 +26,7 @@ jobs: github-token: ${{ github.token }} - name: Comment on PR - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index 6cd7edf89bdb9..282a012192400 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -38,7 +38,7 @@ jobs: steps: # Get the artifacts with the PR number and branch name - name: Download artifact - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); @@ -58,7 +58,7 @@ jobs: # Extract the info from the artifact and set variables - name: Extract PR info from artifact - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); @@ -85,7 +85,7 @@ jobs: # Validate the integrity of the artifact - name: Validate Artifact Integrity - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const crypto = require('crypto'); @@ -104,7 +104,7 @@ jobs: # Kick off the job in amplify - name: Deploy Site - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: APP_ID: ${{ inputs.APP_ID }} APP_NAME: ${{ inputs.APP_NAME }} @@ -171,7 +171,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} APP_ID: ${{ inputs.APP_ID }} APP_NAME: ${{ inputs.APP_NAME }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 3b35ec0b7b9dc..08aea2e9c3c09 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -116,7 +116,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 id: set-matrix with: script: | diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 98c5a6a2df720..fb506c89ead29 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -32,7 +32,7 @@ jobs: # Save PR information (only if branch is valid) - name: Validate and save PR information if: steps.validate.outputs.valid == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs').promises; diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_wip_label.yml index 46d6e79cf1ad8..7c30ee892a949 100644 --- a/.github/workflows/remove_wip_label.yml +++ b/.github/workflows/remove_wip_label.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 5 steps: - name: Check association, label, and remove - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const association = context.payload.review.author_association; From a4ea6de53b829e6ec66fadaddd78317f099207b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:34:31 -0400 Subject: [PATCH 119/364] chore(ci): bump actions/upload-artifact from 7.0.0 to 7.0.1 in the artifact group across 1 directory (#25276) chore(ci): bump actions/upload-artifact Bumps the artifact group with 1 update in the / directory: [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/bbbca2ddaa5d8feaa63e36b76fdaad77386f024f...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: artifact ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/changes.yml | 4 ++-- .github/workflows/check_generated_vrl_docs.yml | 2 +- .github/workflows/cross.yml | 2 +- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/preview_site_trigger.yml | 2 +- .github/workflows/publish.yml | 8 ++++---- .github/workflows/regression.yml | 8 ++++---- .github/workflows/scorecard.yml | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index b07447715d75f..710932c694bc2 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -422,7 +422,7 @@ jobs: echo "any=$any_changed" >> $GITHUB_OUTPUT - name: Upload JSON artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: int_tests_changes path: int_tests_changes.json @@ -484,7 +484,7 @@ jobs: echo "any=$any_changed" >> $GITHUB_OUTPUT - name: Upload JSON artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e_tests_changes path: e2e_tests_changes.json diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index 867ffd7619379..4d30b4f8d5bd2 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -114,7 +114,7 @@ jobs: && steps.last-commit.outputs.is-auto != 'true' && github.event_name == 'pull_request' && steps.push.outcome != 'success' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vrl-docs-check-pr path: /tmp/docs-check/pr-number diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index b554a45d45c09..bc2b2b559503b 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -55,7 +55,7 @@ jobs: # aarch64 and musl in particular are notoriously hard to link. # While it may be tempting to slot a `check` in here for quickness, please don't. - run: make cross-build-${{ matrix.target }} - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: "vector-debug-${{ matrix.target }}" path: "./target/${{ matrix.target }}/debug/vector" diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 08aea2e9c3c09..6b65985cc6eb6 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -95,7 +95,7 @@ jobs: - run: VECTOR_VERSION="$(vdev version)" make package-deb-x86_64-unknown-linux-gnu - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-test-deb-package path: target/artifacts/* diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index fb506c89ead29..db74832c36d60 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -54,7 +54,7 @@ jobs: # Upload the artifact using latest version (only if branch is valid) - name: Upload PR information artifact if: steps.validate.outputs.valid == 'true' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: pr path: pr/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5ccb5ea82c968..4f82bd06e3894 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -81,7 +81,7 @@ jobs: - name: Build Vector run: make package-${{ matrix.target }}-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts/vector* @@ -129,7 +129,7 @@ jobs: export PATH="$HOME/.cargo/bin:$PATH" make package - name: Stage package artifacts for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.architecture }}-apple-darwin path: target/artifacts/vector* @@ -176,7 +176,7 @@ jobs: export PATH="/c/wix:$PATH" ./scripts/package-msi.sh - name: Stage package artifacts for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts/vector* @@ -470,7 +470,7 @@ jobs: - name: Generate SHA256 checksums for artifacts run: make sha256sum - name: Stage checksum for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts/vector-${{ env.VECTOR_VERSION }}-SHA256SUMS diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index aaf71eb9687a4..94b47333a8ed8 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -223,7 +223,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.baseline-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: baseline-image path: "${{ runner.temp }}/baseline-image.tar" @@ -266,7 +266,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.comparison-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: comparison-image path: "${{ runner.temp }}/comparison-image.tar" @@ -433,7 +433,7 @@ jobs: --submission-metadata ${{ runner.temp }}/submission-metadata \ --replicas ${{ env.SMP_REPLICAS }} - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-submission-metadata path: ${{ runner.temp }}/submission-metadata @@ -557,7 +557,7 @@ jobs: path: ${{ runner.temp }}/outputs/report.md - name: Upload regression report to artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: capture-artifacts path: ${{ runner.temp }}/outputs/* diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 391406f824e27..9e64b64ccfabd 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: SARIF file path: results.sarif From 1d4b6c2122b82997b808bba0527ec60dbb37a3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20AGBEKODO?= Date: Mon, 27 Apr 2026 17:36:38 +0200 Subject: [PATCH 120/364] fix(splunk_hec source): Emit warn on unauthenticated request (#25230) * fix(splunk_hec source): Emit warn on unauthenticated request * add changelog * Address comments * fix(splunk_hec source): use authentication_failed error type for unauthenticated requests Add a new `authentication_failed` error_type constant so consumers can filter auth failures from other request errors in `component_errors_total`. * Update changelog.md --------- Co-authored-by: Thomas --- ...plunk_hec_source_specify_wrong_auth.fix.md | 3 ++ .../src/internal_event/prelude.rs | 2 + src/internal_events/splunk_hec.rs | 42 +++++++++++++------ 3 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md diff --git a/changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md b/changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md new file mode 100644 index 0000000000000..8c2af5dd212e4 --- /dev/null +++ b/changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md @@ -0,0 +1,3 @@ +The error log + metric that `splunk_hec` source emit on missing/invalid auth header now specifies "authentication_failed" as error_type. + +authors: 20agbekodo diff --git a/lib/vector-common/src/internal_event/prelude.rs b/lib/vector-common/src/internal_event/prelude.rs index 92aa160fe509e..bcc68aef26c6b 100644 --- a/lib/vector-common/src/internal_event/prelude.rs +++ b/lib/vector-common/src/internal_event/prelude.rs @@ -15,6 +15,8 @@ pub mod error_type { // When a condition for the event to be valid failed. // This is used for example when a field is missing or should be a string. pub const CONDITION_FAILED: &str = "condition_failed"; + // When the component received a request with missing or invalid authentication credentials. + pub const AUTHENTICATION_FAILED: &str = "authentication_failed"; // When the component or the service on which it depends is not configured properly. pub const CONFIGURATION_FAILED: &str = "configuration_failed"; // When the component failed to connect to an external service. diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index ea090684ed85d..6dcd63084d8e6 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -234,18 +234,36 @@ mod source { impl InternalEvent for SplunkHecRequestError { fn emit(self) { - error!( - message = "Error processing request.", - error = ?self.error, - error_type = error_type::REQUEST_FAILED, - stage = error_stage::RECEIVING - ); - counter!( - "component_errors_total", - "error_type" => error_type::REQUEST_FAILED, - "stage" => error_stage::RECEIVING, - ) - .increment(1); + match self.error { + ApiError::InvalidAuthorization | ApiError::MissingAuthorization => { + error!( + message = "Unauthenticated request.", + error = ?self.error, + error_type = error_type::AUTHENTICATION_FAILED, + stage = error_stage::RECEIVING + ); + counter!( + "component_errors_total", + "error_type" => error_type::AUTHENTICATION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + _ => { + error!( + message = "Error processing request.", + error = ?self.error, + error_type = error_type::REQUEST_FAILED, + stage = error_stage::RECEIVING + ); + counter!( + "component_errors_total", + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + } } } } From 156b832637c2f18b4d32fbb347e0e5f0919407d3 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 14:33:20 -0400 Subject: [PATCH 121/364] fix(website): include page title in docs search query fields (#25255) Co-authored-by: Pavlos Rontidis --- website/typesense.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/typesense.config.json b/website/typesense.config.json index 249a303dc895e..9ea772c4c5c12 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -143,7 +143,7 @@ "name": "vector_docs_search", "search_parameters": { "sort_by": "level:desc,ranking:desc,_text_match:desc", - "query_by": "content,hierarchy,title,tags" + "query_by": "pageTitle,title,hierarchy,content,tags" } } ] From a843435f9677fe45f482ff4604bfe92b8a3d57c7 Mon Sep 17 00:00:00 2001 From: Rob Blafford Date: Mon, 27 Apr 2026 16:16:32 -0400 Subject: [PATCH 122/364] fix(topology): Fix for issue causing stalling on shutdown for sinks configured w/ disk buffers (#24949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(buffers): prevent reload stall when disk buffer config changes - Changing a disk buffer's configuration (e.g. `max_size`) during a live config reload would stall indefinitely or fail with a `buffer.lock` error. This was caused by two issues: 1. The sink's detach trigger was only cancelled for buffer-reuse cases, so sinks with changed disk buffer configs never had their input stream terminated — disk buffer readers do not return `None` when the writer disconnects, causing the old sink task to hang forever. 2. The source output pump only processed fanout control messages (Remove/Pause) during active sends, so idle sources would never drop their `BufferSender` clone, keeping the `Arc` file lock alive even after the sink task completed. - Cancel the detach trigger for changed disk buffer sinks so the old sink task can complete. - Make the source output pump `select!` on both events and fanout control messages, so Remove/Pause is processed promptly even when the source is idle. - Add a retry loop (30s timeout) when acquiring the disk buffer lock to handle the small race window between the sink task completing and the fanout releasing the writer. - Add `BufferConfig::has_disk_stage()` helper for identifying disk buffer configurations. * Add unit tests for reload when disk buffer config modified * Add changelog fragment * Integration test exercising full disk buffer path w/ s3 sink - This test sets up a pipeline that sends using the s3 sinks with disk buffers enabled. - Data is sent through the disk buffer before and after configuration reload. * Clean-up changelog fragment * Fix spelling issues * Fix merge error and cleanup use of config() - Make consistent use of the config() method in all tests within this file * Simplify set of control_channel_open status * Remove unnecessary mutable modifiers * Prefer tempDir to just writing to /tmp --------- Co-authored-by: Pavlos Rontidis --- .../24929_fix_stall_on_disk_shutdown.fix.md | 6 + lib/vector-buffers/src/config.rs | 7 + lib/vector-core/src/fanout.rs | 14 ++ src/sinks/aws_s3/integration_tests.rs | 235 ++++++++++++------ src/topology/builder.rs | 62 +++-- src/topology/running.rs | 46 ++-- src/topology/test/reload.rs | 137 +++++++++- 7 files changed, 396 insertions(+), 111 deletions(-) create mode 100644 changelog.d/24929_fix_stall_on_disk_shutdown.fix.md diff --git a/changelog.d/24929_fix_stall_on_disk_shutdown.fix.md b/changelog.d/24929_fix_stall_on_disk_shutdown.fix.md new file mode 100644 index 0000000000000..2c7633caef857 --- /dev/null +++ b/changelog.d/24929_fix_stall_on_disk_shutdown.fix.md @@ -0,0 +1,6 @@ +Fixed issue during in place reload of a sink with a disk buffer configured, where +the component would stall for batch.timeout_sec before gracefully reloading. +This fix also resolves issues Vector had where it would ignore SIGINT during +cases where the pipeline stall had occurred. + +authors: graphcareful diff --git a/lib/vector-buffers/src/config.rs b/lib/vector-buffers/src/config.rs index ea51bfce3cb57..0cc9d77cd29d1 100644 --- a/lib/vector-buffers/src/config.rs +++ b/lib/vector-buffers/src/config.rs @@ -371,6 +371,13 @@ impl Default for BufferConfig { } impl BufferConfig { + /// Returns true if any stage in this buffer configuration uses disk-based storage. + pub fn has_disk_stage(&self) -> bool { + self.stages() + .iter() + .any(|stage| matches!(stage, BufferType::DiskV2 { .. })) + } + /// Gets all of the configured stages for this buffer. pub fn stages(&self) -> &[BufferType] { match self { diff --git a/lib/vector-core/src/fanout.rs b/lib/vector-core/src/fanout.rs index 0fd5cb034b703..97e21bcc8db17 100644 --- a/lib/vector-core/src/fanout.rs +++ b/lib/vector-core/src/fanout.rs @@ -108,6 +108,20 @@ impl Fanout { } } + /// Waits for the next control message and applies it. + /// + /// Returns `true` if a message was processed, `false` if the control + /// channel was closed. + pub async fn recv_control_message(&mut self) -> bool { + match self.control_channel.recv().await { + Some(msg) => { + self.apply_control_message(msg); + true + } + None => false, + } + } + /// Apply a control message directly against this instance. /// /// This method should not be used if there is an active `SendGroup` being processed. diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index ff2e0e5a5146b..a1a2c10af2201 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -1,7 +1,9 @@ #![cfg(all(test, feature = "aws-s3-integration-tests"))] use std::{ + collections::HashSet, io::{BufRead, BufReader}, + num::NonZeroU64, time::Duration, }; @@ -18,12 +20,14 @@ use bytes::Buf; use flate2::read::MultiGzDecoder; use futures::{Stream, stream}; use similar_asserts::assert_eq; +use tempfile::TempDir; use tokio_stream::StreamExt; #[cfg(feature = "codecs-parquet")] use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ + buffers::{BufferConfig, BufferType, WhenFull}, codecs::{TextSerializerConfig, encoding::FramingConfig}, - config::proxy::ProxyConfig, + config::{ComponentKey, proxy::ProxyConfig}, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, EventArray, LogEvent}, }; @@ -31,18 +35,20 @@ use super::S3SinkConfig; use crate::{ aws::{AwsAuthentication, RegionOrEndpoint, create_client}, common::s3::S3ClientBuilder, - config::SinkContext, + config::{Config, SinkContext}, sinks::{ aws_s3::config::default_filename_time_format, s3_common::config::{S3Options, S3ServerSideEncryption}, util::{BatchConfig, Compression, TowerRequestConfig}, }, test_util::{ + self, components::{ AWS_SINK_TAGS, COMPONENT_ERROR_TAGS, run_and_assert_sink_compliance, run_and_assert_sink_error, }, - random_lines_with_stream, random_string, + mock::basic_source, + random_lines_with_stream, random_string, start_topology, }, }; @@ -58,8 +64,10 @@ async fn s3_insert_message_into_with_flat_key_prefix() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix".to_string(), + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -92,8 +100,10 @@ async fn s3_insert_message_into_with_folder_key_prefix() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix/".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix/".to_string(), + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -126,11 +136,16 @@ async fn s3_insert_message_into_with_ssekms_key_id() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix".to_string(), + options: S3Options { + server_side_encryption: Some(S3ServerSideEncryption::AwsKms), + ssekms_key_id: Some("alias/aws/s3".to_string()), + ..S3Options::default() + }, + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); - config.options.server_side_encryption = Some(S3ServerSideEncryption::AwsKms); - config.options.ssekms_key_id = Some("alias/aws/s3".to_string()); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -167,7 +182,7 @@ async fn s3_rotate_files_after_the_buffer_size_is_reached() { key_prefix: format!("{}/{}", random_string(10), "{{i}}"), filename_time_format: "waitsforfullbatch".into(), filename_append_uuid: false, - ..config(&bucket, 10) + ..config(&bucket, 10, 5.0) }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); @@ -225,7 +240,7 @@ async fn s3_gzip() { let config = S3SinkConfig { compression: Compression::gzip_default(), filename_time_format: "%s%f".into(), - ..config(&bucket, batch_size) + ..config(&bucket, batch_size, 5.0) }; let prefix = config.key_prefix.clone(); @@ -270,7 +285,7 @@ async fn s3_zstd() { let config = S3SinkConfig { compression: Compression::zstd_default(), filename_time_format: "%s%f".into(), - ..config(&bucket, batch_size) + ..config(&bucket, batch_size, 5.0) }; let prefix = config.key_prefix.clone(); @@ -334,7 +349,7 @@ async fn s3_insert_message_into_object_lock() { .await .unwrap(); - let config = config(&bucket, 1000000); + let config = config(&bucket, 1000000, 5.0); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -364,9 +379,10 @@ async fn acknowledges_failures() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1); - // Break the bucket name - config.bucket = format!("BREAK{}IT", config.bucket); + let config = S3SinkConfig { + bucket: format!("BREAK{}IT", &bucket), // Break the bucket name + ..config(&bucket, 1, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -385,7 +401,7 @@ async fn s3_healthchecks() { create_bucket(&bucket, false).await; - let config = config(&bucket, 1); + let config = config(&bucket, 1, 5.0); let service = config .create_service(&ProxyConfig::from_env()) .await @@ -399,7 +415,7 @@ async fn s3_healthchecks() { #[tokio::test] async fn s3_healthchecks_invalid_bucket() { - let config = config("s3_healthchecks_invalid_bucket", 1); + let config = config("s3_healthchecks_invalid_bucket", 1, 5.0); let service = config .create_service(&ProxyConfig::from_env()) .await @@ -421,33 +437,7 @@ async fn s3_flush_on_exhaustion() { create_bucket(&bucket, false).await; // batch size of ten events, timeout of ten seconds - let config = { - let mut batch = BatchConfig::default(); - batch.max_events = Some(10); - batch.timeout_secs = Some(10.0); - - S3SinkConfig { - bucket: bucket.to_string(), - key_prefix: random_string(10) + "/date=%F", - filename_time_format: default_filename_time_format(), - filename_append_uuid: true, - filename_extension: None, - options: S3Options::default(), - region: RegionOrEndpoint::with_both("us-east-1", s3_address()), - encoding: (None::, TextSerializerConfig::default()).into(), - #[cfg(feature = "codecs-parquet")] - batch_encoding: None, - compression: Compression::None, - batch, - request: TowerRequestConfig::default(), - tls: Default::default(), - auth: Default::default(), - acknowledgements: Default::default(), - timezone: Default::default(), - force_path_style: true, - retry_strategy: Default::default(), - } - }; + let config = config(&bucket, 10, 10.0); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -511,29 +501,9 @@ async fn s3_parquet_insert_message() { ..Default::default() }; - let mut batch = BatchConfig::default(); - batch.max_events = Some(100); - batch.timeout_secs = Some(5.0); - let config = S3SinkConfig { - bucket: bucket.to_string(), - key_prefix: random_string(10) + "/date=%F", - filename_time_format: default_filename_time_format(), - filename_append_uuid: true, - filename_extension: None, - options: S3Options::default(), - region: RegionOrEndpoint::with_both("us-east-1", s3_address()), - encoding: (None::, TextSerializerConfig::default()).into(), batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), - compression: Compression::None, - batch, - request: TowerRequestConfig::default(), - tls: Default::default(), - auth: Default::default(), - acknowledgements: Default::default(), - timezone: Default::default(), - force_path_style: true, - retry_strategy: Default::default(), + ..config(&bucket, 100, 5.0) }; let prefix = config.key_prefix.clone(); @@ -594,6 +564,131 @@ async fn s3_parquet_insert_message() { assert!(columns.contains(&"host".to_string())); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn s3_disk_buffer_reload_delivers_all_events() { + test_util::trace_init(); + + let bucket = uuid::Uuid::new_v4().to_string(); + create_bucket(&bucket, false).await; + + let data_dir = TempDir::new().expect("Could not create tempdir"); + + // batch.timeout_secs is deliberately large (300 s) so that the test would + // hang if the reload waited for the batch timer instead of cancelling it. + let s3_config = config(&bucket, 10, 300.0); + let prefix = s3_config.key_prefix.clone(); + + // Build topology + let (mut source_tx, source_config) = basic_source(); + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir.path().to_path_buf()); + old_config.add_source("in", source_config); + old_config.add_sink("out", &["in"], s3_config); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }); + + // Clone config before building so we can create the reload config. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }); + + // 1. Start topology with initial disk buffer config. + let (mut topology, _crash) = start_topology(old_config.build().unwrap(), false).await; + + // 2. Send first batch of events (enough to trigger batch.max_events flush). + let (pre_lines, pre_events, pre_receiver) = make_events_batch(100, 10); + for event in pre_events.collect::>().await { + source_tx.send_event(event).await.unwrap(); + } + + // 3. Wait for events to appear in S3. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let keys = get_keys(&bucket, prefix.clone()).await; + let count: usize = futures::future::join_all( + keys.into_iter() + .map(|key| async { get_lines(get_object(&bucket, key).await).await.len() }), + ) + .await + .into_iter() + .sum(); + if count >= pre_lines.len() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "Timed out waiting for pre-reload events in S3 (found {count}/{})", + pre_lines.len() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(pre_receiver.await, BatchStatus::Delivered); + + // 4. Reload config with a different disk buffer max_size. + // Simulate what the `-w` file watcher does: mark the changed sink so its + // buffer is rebuilt rather than reused. + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload timed out: disk buffer config change should not stall the reload" + ); + reload_result.unwrap().unwrap(); + + // Give the new sink a moment to initialise. + tokio::time::sleep(Duration::from_secs(1)).await; + + // 5. Send more events post-reload. + let (post_lines, post_events, post_receiver) = make_events_batch(100, 10); + for event in post_events.collect::>().await { + source_tx.send_event(event).await.unwrap(); + } + + // 6. Assert all events (pre- and post-reload) are present in S3. + let mut all_expected: Vec = pre_lines; + all_expected.extend(post_lines); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let mut all_actual: Vec; + loop { + all_actual = Vec::new(); + let keys = get_keys(&bucket, prefix.clone()).await; + for key in keys { + all_actual.extend(get_lines(get_object(&bucket, key).await).await); + } + if all_actual.len() >= all_expected.len() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "Timed out waiting for all events in S3 (found {}/{})", + all_actual.len(), + all_expected.len() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(post_receiver.await, BatchStatus::Delivered); + + all_expected.sort(); + all_actual.sort(); + assert_eq!(all_expected, all_actual); + + topology.stop().await; +} + async fn client() -> S3Client { let auth = AwsAuthentication::test_auth(); let region = RegionOrEndpoint::with_both("us-east-1", s3_address()); @@ -616,10 +711,10 @@ async fn client() -> S3Client { .unwrap() } -fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { +fn config(bucket: &str, batch_size: usize, timeout_secs: f64) -> S3SinkConfig { let mut batch = BatchConfig::default(); batch.max_events = Some(batch_size); - batch.timeout_secs = Some(5.0); + batch.timeout_secs = Some(timeout_secs); S3SinkConfig { bucket: bucket.to_string(), diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 8048b7a6073ed..8948109fa7fb7 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -884,30 +884,44 @@ async fn run_source_output_pump( ) -> TaskResult { debug!("Source pump starting."); - while let Some(SourceSenderItem { - events: mut array, - send_reference, - }) = rx.next().await - { - // Even though we have a `send_reference` timestamp above, that reference time is when - // the events were enqueued in the `SourceSender`, not when they were pulled out of the - // `rx` stream on this end. Since those times can be quite different (due to blocking - // inherent to the fanout send operation), we set the `last_transform_timestamp` to the - // current time instead to get an accurate reference for when the events started waiting - // for the first transform. - let now = Instant::now(); - array.for_each_metadata_mut(|metadata| { - metadata.set_source_id(Arc::clone(&source)); - metadata.set_source_type(source_type); - metadata.set_last_transform_timestamp(now); - }); - fanout - .send(array, Some(send_reference)) - .await - .map_err(|e| { - debug!("Source pump finished with an error."); - TaskError::wrapped(e) - })?; + let mut control_channel_open = true; + loop { + tokio::select! { + biased; + // Process control messages (e.g. Remove/Pause) even when the source + // is idle, so that config reloads can proceed without waiting for the + // next event. + alive = fanout.recv_control_message(), if control_channel_open => { + control_channel_open = alive; + } + item = rx.next() => { + match item { + Some(SourceSenderItem { events: mut array, send_reference }) => { + // Even though we have a `send_reference` timestamp above, that reference + // time is when the events were enqueued in the `SourceSender`, not when + // they were pulled out of the `rx` stream on this end. Since those times + // can be quite different (due to blocking inherent to the fanout send + // operation), we set the `last_transform_timestamp` to the current time + // instead to get an accurate reference for when the events started + // waiting for the first transform. + let now = Instant::now(); + array.for_each_metadata_mut(|metadata| { + metadata.set_source_id(Arc::clone(&source)); + metadata.set_source_type(source_type); + metadata.set_last_transform_timestamp(now); + }); + fanout + .send(array, Some(send_reference)) + .await + .map_err(|e| { + debug!("Source pump finished with an error."); + TaskError::wrapped(e) + })?; + } + None => break, + } + } + } } debug!("Source pump finished normally."); diff --git a/src/topology/running.rs b/src/topology/running.rs index a1ecf7278ae89..97e309ec0d0b8 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -583,10 +583,26 @@ impl RunningTopology { .collect::>(); // For any existing sink that has a conflicting resource dependency with a changed/added - // sink, or for any sink that we want to reuse their buffer, we need to explicit wait for - // them to finish processing so we can reclaim ownership of those resources/buffers. + // sink, for any sink that we want to reuse their buffer, or for any changed sink with + // a disk buffer that is not being reused, we need to explicitly wait for them to finish + // processing so we can reclaim ownership of those resources/buffers. + let changed_disk_buffer_sinks = diff + .sinks + .to_change + .iter() + .filter(|key| { + !reuse_buffers.contains(*key) + && self + .config + .sink(key) + .is_some_and(|s| s.buffer.has_disk_stage()) + }) + .cloned() + .collect::>(); + let wait_for_sinks = conflicting_sinks .chain(reuse_buffers.iter().cloned()) + .chain(changed_disk_buffer_sinks.iter().cloned()) .collect::>(); // First, we remove any inputs to removed sinks so they can naturally shut down. @@ -628,24 +644,26 @@ impl RunningTopology { for key in &sinks_to_change { debug!(component_id = %key, "Changing sink."); - if reuse_buffers.contains(key) { + if reuse_buffers.contains(key) || changed_disk_buffer_sinks.contains(key) { self.detach_triggers .remove(key) .unwrap() .into_inner() .cancel(); - // We explicitly clone the input side of the buffer and store it so we don't lose - // it when we remove the inputs below. - // - // We clone instead of removing here because otherwise the input will be missing for - // the rest of the reload process, which violates the assumption that all previous - // inputs for components not being removed are still available. It's simpler to - // allow the "old" input to stick around and be replaced (even though that's - // basically a no-op since we're reusing the same buffer) than it is to pass around - // info about which sinks are having their buffers reused and treat them differently - // at other stages. - buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); + if reuse_buffers.contains(key) { + // We explicitly clone the input side of the buffer and store it so we don't lose + // it when we remove the inputs below. + // + // We clone instead of removing here because otherwise the input will be missing for + // the rest of the reload process, which violates the assumption that all previous + // inputs for components not being removed are still available. It's simpler to + // allow the "old" input to stick around and be replaced (even though that's + // basically a no-op since we're reusing the same buffer) than it is to pass around + // info about which sinks are having their buffers reused and treat them differently + // at other stages. + buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); + } } self.remove_inputs(key, diff, new_config).await; } diff --git a/src/topology/test/reload.rs b/src/topology/test/reload.rs index 907be19f6ec0f..7b2db20d4f337 100644 --- a/src/topology/test/reload.rs +++ b/src/topology/test/reload.rs @@ -1,7 +1,7 @@ use std::{ collections::HashSet, net::{SocketAddr, TcpListener}, - num::NonZeroU64, + num::{NonZeroU64, NonZeroUsize}, time::Duration, }; @@ -9,7 +9,7 @@ use futures::StreamExt; use tokio::time::sleep; use tokio_stream::wrappers::UnboundedReceiverStream; use vector_lib::{ - buffers::{BufferConfig, BufferType, WhenFull}, + buffers::{BufferConfig, BufferType, MemoryBufferSize, WhenFull}, config::ComponentKey, }; @@ -293,7 +293,6 @@ async fn topology_readd_input() { #[tokio::test] async fn topology_reload_component() { test_util::trace_init(); - let (_guard, address_0) = next_addr(); let mut old_config = Config::builder(); @@ -320,6 +319,138 @@ async fn topology_reload_component() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn topology_disk_buffer_config_change_does_not_stall() { + // Changing a disk buffer's configuration on a running sink (e.g. via in-situ + // config edit) must not stall the reload. Previously, the detach trigger was + // only cancelled for sinks whose buffers were being reused, so sinks with + // changed disk buffer configs would never have their input stream terminated, + // causing the reload to hang indefinitely. + test_util::trace_init(); + + let (_guard, address) = next_addr(); + + let data_dir = temp_dir(); + std::fs::create_dir(&data_dir).unwrap(); + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir); + old_config.add_source("in", internal_metrics_source()); + old_config.add_sink("out", &["in"], prom_exporter_sink(address, 1)); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }); + + // Change only the disk buffer's max_size. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }); + + let (mut topology, crash) = start_topology(old_config.build().unwrap(), true).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); + + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed before reload"), + } + + // Simulate an in-situ config edit: the config watcher would put the changed + // sink into components_to_reload, which excludes it from reuse_buffers. + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload stalled: changing a disk buffer config should not cause the reload to hang" + ); + reload_result.unwrap().unwrap(); + + // Verify the new sink is running. + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed after reload"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn topology_disk_buffer_config_change_chained_does_not_stall() { + // Same as above but with a chained memory → disk overflow buffer to verify + // that the writer-drop notification is collected from overflow stages too. + test_util::trace_init(); + + let (_guard, address) = next_addr(); + + let data_dir = temp_dir(); + std::fs::create_dir(&data_dir).unwrap(); + + let memory_stage = BufferType::Memory { + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), + when_full: WhenFull::Overflow, + }; + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir); + old_config.add_source("in", internal_metrics_source()); + old_config.add_sink("out", &["in"], prom_exporter_sink(address, 1)); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Chained(vec![ + memory_stage, + BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }, + ]); + + // Change only the disk overflow stage's max_size. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Chained(vec![ + memory_stage, + BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }, + ]); + + let (mut topology, crash) = start_topology(old_config.build().unwrap(), true).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); + + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed before reload"), + } + + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload stalled: changing a chained disk buffer config should not cause the reload to hang" + ); + reload_result.unwrap().unwrap(); + + // Verify the new sink is running. + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed after reload"), + } +} + async fn reload_sink_test( old_config: Config, new_config: Config, From ff4754ab10da5838f68cc143e507087090518f91 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 10:04:28 -0400 Subject: [PATCH 123/364] chore(ci): make nightly S3 verify resilient to CDN staleness (#25259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ci): remove dead nightly artifact-redirect loop The loop referenced an undefined $i (copy-paste from the release branch where $i iterates over version tags). Under set -u the subshell errored, the for loop received empty input, and the body never ran — so this has been a no-op since it was added. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): retry verify_artifact on CDN cache staleness packages.timber.io is fronted by a CDN; after `aws s3 rm --recursive` + `cp --recursive` on nightly/latest, the edge can keep serving stale bytes for longer than the existing 30s VERIFY_TIMEOUT, failing `cmp` and the whole job. wget's --retry-on-http-error=404 only retries 404s, not a 200 with stale content. Wrap the compare in an exponential-backoff retry loop (1, 2, 4, 8, 16, 32s; 7 attempts, ~63s of total sleep) so a transient stale-cache hit no longer fails the release. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- scripts/release-s3.sh | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/release-s3.sh b/scripts/release-s3.sh index 9889fa556e4eb..8f6c51de08669 100755 --- a/scripts/release-s3.sh +++ b/scripts/release-s3.sh @@ -36,11 +36,27 @@ ls "$td_nightly" # # A helper function for verifying a published artifact. # +# Retries a content mismatch as well as a 404, since packages.timber.io is +# fronted by a CDN and an object we just overwrote via `aws s3 rm` + `cp` can +# serve stale bytes at the edge for a while. verify_artifact() { local URL="$1" local FILENAME="$2" + local attempts=7 + local delay=1 echo "Verifying $URL" - cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME" + for ((attempt = 1; attempt <= attempts; attempt++)); do + if cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME"; then + return 0 + fi + if (( attempt < attempts )); then + echo "Attempt $attempt/$attempts did not match (likely stale CDN cache); retrying in ${delay}s" + sleep "$delay" + delay=$((delay * 2)) + fi + done + echo "Verification of $URL failed after $attempts attempts" + return 1 } # @@ -59,15 +75,6 @@ if [[ "$CHANNEL" == "nightly" ]]; then aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/latest" --recursive --sse --acl public-read echo "Uploaded archives" - echo "Redirecting old artifact names" - for file in $(aws s3api list-objects-v2 --bucket packages.timber.io --prefix "vector/$i/" --query 'Contents[*].Key' --output text | tr "\t" "\n" | grep '\-nightly'); do - file=$(basename "$file") - # vector-nightly-amd64.deb -> vector-amd64.deb - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/$DATE/${file/-nightly/}" --website-redirect "/vector/nightly/$DATE/$file" --acl public-read - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/latest/${file/-nightly/}" --website-redirect "/vector/nightly/latest/$file" --acl public-read - done - echo "Redirected old artifact names" - # Verify that the files exist and can be downloaded echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" sleep "$VERIFY_TIMEOUT" From 6f3857906d3a54d9c05aa5d3aaa8b132ac8f5e6e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 11:12:07 -0400 Subject: [PATCH 124/364] fix(vrl): restore stdlib functions in CLI and playground (#25310) --- changelog.d/25267_vrl_get_env_var_playground.fix.md | 2 ++ lib/vector-vrl/cli/Cargo.toml | 4 ++++ lib/vector-vrl/web-playground/Cargo.toml | 4 ++++ 3 files changed, 10 insertions(+) create mode 100644 changelog.d/25267_vrl_get_env_var_playground.fix.md diff --git a/changelog.d/25267_vrl_get_env_var_playground.fix.md b/changelog.d/25267_vrl_get_env_var_playground.fix.md new file mode 100644 index 0000000000000..29de5d36fe6a5 --- /dev/null +++ b/changelog.d/25267_vrl_get_env_var_playground.fix.md @@ -0,0 +1,2 @@ +Restored the full VRL stdlib, including `get_env_var`, in the standalone VRL CLI and web playground by default. +authors: pront diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 32f0b8ccce166..8e3c78597de8b 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -6,6 +6,10 @@ edition = "2024" publish = false license = "MPL-2.0" +[features] +# Enable the full VRL stdlib, which includes all available VRL functions. +default = ["vrl/stdlib"] + [dependencies] clap.workspace = true vector-vrl-functions.workspace = true diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 9b3fbd6392576..59db0427b3698 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -11,6 +11,10 @@ license = "MPL-2.0" [lib] crate-type = ["cdylib"] +[features] +# Enable the full VRL stdlib, which includes all available VRL functions. +default = ["vrl/stdlib"] + [dependencies] wasm-bindgen = "0.2" vrl.workspace = true From 23016adf7d87f41a3d887d587b21d3409e71e3fe Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 11:52:16 -0400 Subject: [PATCH 125/364] chore(internal docs): fix release issue templates (#25318) - Fetch and fast-forward the release branch before inspecting HEAD so `git show --stat HEAD` reflects origin, not a stale local tip. - Use `git push --force-with-lease` when resetting the `website` branch to the release branch's HEAD; a plain `git push` is rejected as non-fast-forward, which is the expected outcome of `reset --hard` to a different branch. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/ISSUE_TEMPLATE/minor-release.md | 5 +++-- .github/ISSUE_TEMPLATE/patch-release.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 7ab406601e4cb..f295a03555512 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -71,7 +71,8 @@ Automated steps include: # On the day of release - [ ] Make sure the release branch is in sync with origin/master and has only one squashed commit with all commits from the prepare branch. If you made a PR from the prepare branch into the release branch this should already be the case. - - [ ] `git checkout "${RELEASE_BRANCH}"` + - [ ] `git fetch origin` + - [ ] `git checkout "${RELEASE_BRANCH}" && git pull --ff-only origin "${RELEASE_BRANCH}"` - [ ] `git show --stat HEAD` - This should show the squashed prepare commit. - [ ] Ensure release date in `website/cue/reference/releases/0.XX.Y.cue` matches current date. - If this needs to be updated commit and squash it in the release branch. @@ -88,7 +89,7 @@ Automated steps include: - [ ] Wait for release workflow to complete. - Discoverable via [release.yml](https://github.com/vectordotdev/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` + - [ ] `git fetch origin && git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push --force-with-lease` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - Refer to the internal releasing doc to monitor the deployment. - [ ] Release Linux packages. Refer to the internal releasing doc. diff --git a/.github/ISSUE_TEMPLATE/patch-release.md b/.github/ISSUE_TEMPLATE/patch-release.md index afc5d8d826423..4a36f7255e36b 100644 --- a/.github/ISSUE_TEMPLATE/patch-release.md +++ b/.github/ISSUE_TEMPLATE/patch-release.md @@ -62,5 +62,5 @@ export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" - Follow the [instructions at the top of the mirror.yaml file](https://github.com/DataDog/images/blob/fbf12868e90d52e513ebca0389610dea8a3c7e1a/mirror.yaml#L33-L49). - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` + - [ ] `git fetch origin && git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push --force-with-lease` - [ ] Kick-off post-mortems for any regressions resolved by the release From 96ad9edc5bd894029af95961ea205e0d89b17bf0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 28 Apr 2026 14:17:38 -0400 Subject: [PATCH 126/364] fix(website): improve docs search ranking for component pages (#25319) --- website/typesense.config.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/website/typesense.config.json b/website/typesense.config.json index 9ea772c4c5c12..f0543b9133c8b 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -5,6 +5,7 @@ "file_path": "./public/search.json", "synonyms": [], "schema": { + "token_separators": ["_", "-"], "fields": [ { "facet": false, @@ -142,8 +143,10 @@ { "name": "vector_docs_search", "search_parameters": { - "sort_by": "level:desc,ranking:desc,_text_match:desc", - "query_by": "pageTitle,title,hierarchy,content,tags" + "sort_by": "_text_match:desc,ranking:desc,level:desc", + "query_by": "pageTitle,title,hierarchy,content,tags", + "query_by_weights": "4,3,2,1,2", + "text_match_type": "sum_score" } } ] From ce6ca439caf80bd2af10864a84e949ef244e2163 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 15:40:29 -0400 Subject: [PATCH 127/364] chore(codecs): centralize `events_dropped` emission for batch encoding errors (#25199) * fix(codecs): centralize events_dropped emission for batch encoding errors Move events_dropped emission from individual internal events inside serializers to a single wrapper in (Transformer, BatchEncoder)::encode_input. This ensures all batch encoding error paths (Arrow IPC and Parquet) consistently emit events_dropped without requiring each new error path to remember to add it. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: fix formatting Co-Authored-By: Claude Opus 4.6 (1M context) * test(codecs): add unit test for type mismatch in Parquet encoding Covers the build_record_batch ArrowJsonDecode error path where a schema expects int64 but the event contains a string value. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(codecs): emit ComponentEventsDropped directly to avoid double-counting Replace EncoderWriteError with a direct ComponentEventsDropped emission in the batch encode wrapper. EncoderWriteError was incrementing component_errors_total and logging "Failed writing bytes." which double-counted errors (codec-specific events already increment component_errors_total) and was misleading (the failure was encoding, not writing). Co-Authored-By: Claude Opus 4.6 (1M context) * docs: clarify comment about error counting and overcount edge case Co-Authored-By: Claude Opus 4.6 (1M context) * fix(codecs): guard batch-drop imports behind codecs-arrow feature Move ComponentEventsDropped and UNINTENTIONAL imports inside the cfg(feature = "codecs-arrow") impl block to avoid unused import errors when the feature is disabled. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: clarify overcount edge case is a misconfiguration scenario Co-Authored-By: Claude Opus 4.6 (1M context) * chore(codecs): emit codec event for build_record_batch failures The new EncoderRecordBatchError fires from build_record_batch's RecordBatchCreation and ArrowJsonDecode paths, so type-mismatch and decoder-build failures emit a granular component_errors_total counter at stage="sending" with a specific error_code, instead of relying solely on the downstream SinkRequestBuildError at stage="processing". Co-Authored-By: Claude Opus 4.7 (1M context) * chore(codecs): add changelog fragment for batch encoding event coverage Co-Authored-By: Claude Opus 4.7 (1M context) * test(codecs): assert EncoderRecordBatchError fires on Arrow type mismatch Drives (Transformer, BatchEncoder)::encode_input through ArrowStreamSerializer with an Int64 schema field and a string-valued event to trigger the ArrowJsonDecode path in build_record_batch. Asserts both EncoderRecordBatchError and ComponentEventsDropped are recorded so a regression in either emission fails the test. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...25199_batch_encoding_events_dropped.fix.md | 3 + lib/codecs/src/encoding/format/arrow.rs | 23 ++++++- lib/codecs/src/encoding/format/parquet.rs | 63 +++++++++--------- lib/codecs/src/internal_events.rs | 64 ++++++++++--------- src/sinks/util/encoding.rs | 61 +++++++++++++++++- 5 files changed, 153 insertions(+), 61 deletions(-) create mode 100644 changelog.d/25199_batch_encoding_events_dropped.fix.md diff --git a/changelog.d/25199_batch_encoding_events_dropped.fix.md b/changelog.d/25199_batch_encoding_events_dropped.fix.md new file mode 100644 index 0000000000000..214aad79a528c --- /dev/null +++ b/changelog.d/25199_batch_encoding_events_dropped.fix.md @@ -0,0 +1,3 @@ +Sinks using batch encoding (Parquet, Arrow IPC) now consistently emit `ComponentEventsDropped` for every encode failure path. Previously some `build_record_batch` failures (notably type mismatches) dropped events silently. A new `EncoderRecordBatchError` internal event also reports `component_errors_total` with `error_code="arrow_json_decode"` or `"arrow_record_batch_creation"` at `stage="sending"` for granular alerting. + +authors: pront diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index b8c86239ddb6a..6c600b3777d18 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -322,7 +322,6 @@ pub(crate) fn build_record_batch( }); vector_common::internal_event::emit(crate::internal_events::EncoderNullConstraintError { error: &error, - count: values.len(), }); return Err(ArrowEncodingError::NullConstraint { field_name: missing.join(", "), @@ -331,12 +330,32 @@ pub(crate) fn build_record_batch( let mut decoder = ReaderBuilder::new(schema) .build_decoder() + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_record_batch_creation", + }); + }) .context(RecordBatchCreationSnafu)?; - decoder.serialize(values).context(ArrowJsonDecodeSnafu)?; + decoder + .serialize(values) + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_json_decode", + }); + }) + .context(ArrowJsonDecodeSnafu)?; decoder .flush() + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_json_decode", + }); + }) .context(ArrowJsonDecodeSnafu)? .ok_or(ArrowEncodingError::NoEvents) } diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs index 7349119371a5b..adb5b4082749b 100644 --- a/lib/codecs/src/encoding/format/parquet.rs +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -276,39 +276,29 @@ impl ParquetSerializer { /// Writes `record_batch` into `buffer` as a complete Parquet file. /// - /// On failure, emits an [`ArrowWriterError`] internal event (which also - /// increments `component_errors_total` and emits the events-dropped metric) - /// before returning the error. + /// On failure, emits an [`ArrowWriterError`] internal event (which + /// increments `component_errors_total`) before returning the error. + /// The caller is responsible for emitting `events_dropped`. fn write_record_batch( - &self, record_batch: &RecordBatch, buffer: &mut BytesMut, - event_count: usize, + writer_props: &WriterProperties, ) -> Result<(), parquet::errors::ParquetError> { let mut writer = ArrowWriter::try_new( buffer.writer(), Arc::clone(record_batch.schema_ref()), - Some((*self.writer_props).clone()), + Some(writer_props.clone()), ) .inspect_err(|e| { - emit(ArrowWriterError { - error: e, - batch_count: event_count, - }); + emit(ArrowWriterError { error: e }); })?; writer.write(record_batch).inspect_err(|e| { - emit(ArrowWriterError { - error: e, - batch_count: event_count, - }); + emit(ArrowWriterError { error: e }); })?; writer.close().inspect_err(|e| { - emit(ArrowWriterError { - error: e, - batch_count: event_count, - }); + emit(ArrowWriterError { error: e }); })?; Ok(()) @@ -326,10 +316,7 @@ impl tokio_util::codec::Encoder> for ParquetSerializer { let json_values = match vector_log_events_to_json_values(&events) { Ok(values) => values, Err(e) => { - emit(JsonSerializationError { - error: &e, - batch_count: events.len(), - }); + emit(JsonSerializationError { error: &e }); return Err(Box::new(e)); } }; @@ -358,7 +345,6 @@ impl tokio_util::codec::Encoder> for ParquetSerializer { { for top_level in object_map.keys() { if !self.schema_field_names.contains(top_level.as_str()) { - self.events_dropped_handle.emit(Count(events.len())); return Err(Box::new(ArrowEncodingError::SchemaFetchError { message: format!( "Strict schema mode: event contains field '{top_level}' not in schema", @@ -381,8 +367,7 @@ impl tokio_util::codec::Encoder> for ParquetSerializer { let record_batch = build_record_batch(Arc::clone(&self.schema), &json_values).map_err(Box::new)?; - self.write_record_batch(&record_batch, buffer, json_values.len()) - .map_err(Box::new)?; + Self::write_record_batch(&record_batch, buffer, &self.writer_props).map_err(Box::new)?; Ok(()) } @@ -394,10 +379,7 @@ impl ParquetSchemaGenerator { pub fn infer_schema(events: &[serde_json::Value]) -> Result { let schema = infer_json_schema_from_iterator(events.iter().map(Ok::<_, ArrowError>)) .map_err(|e| { - emit(SchemaGenerationError { - error: &e, - batch_count: events.len(), - }); + emit(SchemaGenerationError { error: &e }); Error::new(ErrorKind::InvalidData, e.to_string()) })?; @@ -916,6 +898,29 @@ mod tests { assert_eq!(columns, vec!["name"]); } + #[test] + fn test_parquet_type_mismatch_returns_error() { + let schema_path = + write_temp_schema("type_mismatch", "message logs {\n required int64 name;\n}"); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Relaxed, + ..Default::default() + }) + .expect("Failed to create serializer"); + + let events = vec![create_event(vec![("name", "not_an_integer")])]; + let mut buffer = BytesMut::new(); + let result = serializer.encode(events, &mut buffer); + assert!(result.is_err(), "Type mismatch should return an error"); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("Int64"), + "Error should mention the expected type, got: {err}" + ); + } + #[test] fn test_parquet_schema_file_binary_without_string_annotation_rejected() { // Native Parquet "binary" without (STRING) annotation resolves to Arrow Binary, diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index 0ed24719dd8bc..d21119419887d 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -157,16 +157,13 @@ impl InternalEvent for EncoderWriteError<'_, E> { pub struct EncoderNullConstraintError<'a> { /// The schema constraint error that occurred. pub error: &'a vector_common::Error, - /// The number of events dropped due to the constraint violation. - pub count: usize, } #[cfg(feature = "arrow")] impl InternalEvent for EncoderNullConstraintError<'_> { fn emit(self) { - const CONSTRAINT_REASON: &str = "Schema constraint violation."; error!( - message = CONSTRAINT_REASON, + message = "Schema constraint violation.", error = %self.error, error_code = "encoding_null_constraint", error_type = error_type::ENCODER_FAILED, @@ -179,10 +176,37 @@ impl InternalEvent for EncoderNullConstraintError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.count, - reason: CONSTRAINT_REASON, - }); + } +} + +#[cfg(feature = "arrow")] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when Arrow record batch construction fails (e.g. schema decoder build, +/// JSON-to-Arrow decoding such as type mismatches). +pub struct EncoderRecordBatchError<'a, E> { + /// The encoding error that occurred. + pub error: &'a E, + /// Stable error code identifying the failure mode. + pub error_code: &'static str, +} + +#[cfg(feature = "arrow")] +impl InternalEvent for EncoderRecordBatchError<'_, E> { + fn emit(self) { + error!( + message = "Failed to build Arrow record batch.", + error = %self.error, + error_code = self.error_code, + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + ); + counter!( + "component_errors_total", + "error_code" => self.error_code, + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); } } @@ -190,15 +214,13 @@ impl InternalEvent for EncoderNullConstraintError<'_> { #[derive(NamedInternalEvent)] pub(crate) struct SchemaGenerationError<'a> { pub error: &'a arrow::error::ArrowError, - pub batch_count: usize, } #[cfg(feature = "parquet")] impl InternalEvent for SchemaGenerationError<'_> { fn emit(self) { - const REASON: &str = "Could not generate schema for batched events"; error!( - message = REASON, + message = "Could not generate schema for batched events", error = %self.error, error_code = "parquet_schema_generation_failed", error_type = error_type::ENCODER_FAILED, @@ -212,10 +234,6 @@ impl InternalEvent for SchemaGenerationError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.batch_count, - reason: REASON, - }); } } @@ -223,15 +241,13 @@ impl InternalEvent for SchemaGenerationError<'_> { #[derive(NamedInternalEvent)] pub(crate) struct ArrowWriterError<'a> { pub error: &'a parquet::errors::ParquetError, - pub batch_count: usize, } #[cfg(feature = "parquet")] impl InternalEvent for ArrowWriterError<'_> { fn emit(self) { - const REASON: &str = "Failed to write record batch with ArrowWriter."; error!( - message = REASON, + message = "Failed to write record batch with ArrowWriter.", error = %self.error, error_code = "parquet_arrow_writer_failed", error_type = error_type::ENCODER_FAILED, @@ -245,10 +261,6 @@ impl InternalEvent for ArrowWriterError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.batch_count, - reason: REASON, - }); } } @@ -256,15 +268,13 @@ impl InternalEvent for ArrowWriterError<'_> { #[derive(NamedInternalEvent)] pub(crate) struct JsonSerializationError<'a> { pub error: &'a serde_json::Error, - pub batch_count: usize, } #[cfg(feature = "parquet")] impl InternalEvent for JsonSerializationError<'_> { fn emit(self) { - const CONSTRAINT_REASON: &str = "Could not serialize event to JSON."; error!( - message = CONSTRAINT_REASON, + message = "Could not serialize event to JSON.", error = %self.error, error_type = error_type::ENCODER_FAILED, stage = error_stage::SENDING, @@ -276,9 +286,5 @@ impl InternalEvent for JsonSerializationError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.batch_count, - reason: CONSTRAINT_REASON, - }); } } diff --git a/src/sinks/util/encoding.rs b/src/sinks/util/encoding.rs index 4cc49b00f358b..63f8407cef53b 100644 --- a/src/sinks/util/encoding.rs +++ b/src/sinks/util/encoding.rs @@ -107,6 +107,7 @@ impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { writer: &mut dyn io::Write, ) -> io::Result<(usize, GroupedCountByteSize)> { use tokio_util::codec::Encoder as _; + use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL}; let mut encoder = self.1.clone(); let mut byte_size = telemetry().create_request_count_byte_size(); @@ -122,7 +123,22 @@ impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { let mut bytes = BytesMut::new(); encoder .encode(transformed_events, &mut bytes) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + .map_err(|error| { + // Codec error paths emit their own internal event + // (e.g. SchemaGenerationError, EncoderNullConstraintError, + // EncoderRecordBatchError) which logs the error and increments + // component_errors_total. We only emit the drop count here to + // avoid double-counting. + // n_events is the pre-filter count; Parquet filters non-log + // events before encoding, but that only happens if a sink is + // misconfigured to send non-log events into a log-only encoder, + // so the overcount is not a practical concern. + emit!(ComponentEventsDropped:: { + count: n_events, + reason: "Failed to batch encode events.", + }); + io::Error::new(io::ErrorKind::InvalidData, error) + })?; write_all(writer, n_events, &bytes)?; Ok((bytes.len(), byte_size)) @@ -202,6 +218,7 @@ mod tests { use std::{collections::BTreeMap, env, path::PathBuf}; use bytes::{BufMut, Bytes}; + use cfg_if::cfg_if; use vector_lib::{ codecs::{ CharacterDelimitedEncoder, JsonSerializerConfig, LengthDelimitedEncoder, @@ -214,6 +231,17 @@ mod tests { }; use vrl::value::{KeyString, Value}; + cfg_if! { + if #[cfg(feature = "codecs-arrow")] { + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use vector_lib::codecs::{ + BatchEncoder, + encoding::{ArrowStreamSerializer, ArrowStreamSerializerConfig, BatchSerializer}, + }; + use vector_lib::event_test_util::{clear_recorded_events, contains_name_once}; + } + } + use super::*; #[test] @@ -558,4 +586,35 @@ mod tests { assert_eq!(CountByteSize(2, input_json_size), size.size().unwrap()); assert_eq!(Bytes::copy_from_slice(&writer), expected_bytes); } + + #[cfg(feature = "codecs-arrow")] + #[test] + fn test_encode_batch_arrow_emits_record_batch_error_on_type_mismatch() { + clear_recorded_events(); + + // Schema declares `message` as Int64, but the event below carries a string, + // so `build_record_batch` returns `ArrowEncodingError::ArrowJsonDecode`. + let schema = ArrowSchema::new(vec![Field::new("message", DataType::Int64, false)]); + let serializer = ArrowStreamSerializer::new(ArrowStreamSerializerConfig::new(schema)) + .expect("failed to build ArrowStreamSerializer"); + let encoder = BatchEncoder::new(BatchSerializer::Arrow(serializer)); + let encoding = (Transformer::default(), encoder); + + let event = Event::Log(LogEvent::from(BTreeMap::from([( + KeyString::from("message"), + Value::from("not_an_integer"), + )]))); + + let mut writer = Vec::new(); + let result = encoding.encode_input(vec![event], &mut writer); + assert!( + result.is_err(), + "type mismatch should fail batch encoding, got {result:?}" + ); + + contains_name_once("EncoderRecordBatchError") + .expect("EncoderRecordBatchError should be emitted on ArrowJsonDecode failure"); + contains_name_once("ComponentEventsDropped") + .expect("ComponentEventsDropped should be emitted by the wrapper"); + } } From d6cdf031d16a382a38046127fbc7ff30c2457709 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 17:40:45 -0400 Subject: [PATCH 128/364] fix(releasing): enable codecs-parquet in all release feature sets (#25321) * fix(releasing): enable codecs-parquet in all release feature sets The aws_s3 sink's `batch_encoding` field is gated behind the `codecs-parquet` Cargo feature, but the v0.55.0 release artifacts did not include that feature. Users running the precompiled binaries hit `unknown field batch_encoding` even though the feature was advertised in the v0.55.0 release notes. Enable `codecs-parquet` in `target-base` (covers every Linux release triple), `default` (macOS), and `default-msvc` (Windows), plus the related aggregator features for consistency. Closes #25313 Co-Authored-By: Claude Opus 4.7 (1M context) * fix(releasing): correct changelog author handle Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): trim changelog wording Co-Authored-By: Claude Opus 4.7 (1M context) * chore(external docs): regenerate component docs with parquet codec Adds the missing `docs::enum_tag_description` metadata on `ParquetCompression` (required by the doc generator for internally tagged enums) and regenerates the aws_s3 and clickhouse component Cue files now that `codecs-parquet` and `codecs-arrow` are part of the default release feature set. Co-Authored-By: Claude Opus 4.7 (1M context) * docs: address review wording on batch encoding fields Apply @maycmlee's suggestions in the source rustdocs (and regenerate the affected Cue): - aws_s3 `batch_encoding`: `e.g., Parquet` -> `for example, Parquet` - arrow `allow_nullable_fields`: tighter wording for both enabled and disabled cases - parquet `Zstd`/`Gzip` `level`: drop "currently" from "Vector currently supports" Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(releasing): consolidate codecs-parquet into base feature Every release feature aggregator (default, default-cmake, default-msvc, default-musl, default-no-api-client, target-base) transitively pulls `base` either directly or via `enable-api-client`/`enable-unix`. Drop the redundant per-aggregator listings and add `codecs-parquet` to `base` once so the feature can't be silently lost when a future aggregator gets added. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(aws_s3, clickhouse): note supported batch_encoding codec per sink The `batch_encoding` field on both sinks types as `Option`, which advertises every codec variant in the generated reference. Each sink only accepts a subset at runtime: aws_s3 rejects everything except `parquet`, clickhouse rejects `parquet`. Call out the supported codec in the field's rustdoc so the published reference matches what the sink will actually accept. A follow-up should replace the shared enum with sink-specific config types so the type system enforces the constraint. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- Cargo.toml | 2 +- ...3_enable_codecs_parquet_in_releases.fix.md | 3 + lib/codecs/src/encoding/format/arrow.rs | 6 +- lib/codecs/src/encoding/format/parquet.rs | 7 +- src/sinks/aws_s3/config.rs | 4 +- src/sinks/clickhouse/config.rs | 2 + .../components/sinks/generated/aws_s3.cue | 99 +++++++++++++++++++ .../components/sinks/generated/clickhouse.cue | 86 +++++++++++++--- 8 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 changelog.d/25313_enable_codecs_parquet_in_releases.fix.md diff --git a/Cargo.toml b/Cargo.toml index 70f9fc12a8acc..52161db4b2905 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -519,7 +519,7 @@ vendored = ["rdkafka?/gssapi-vendored"] # Default features for *-pc-windows-msvc # TODO: Enable SASL https://github.com/vectordotdev/vector/pull/3081#issuecomment-659298042 -base = ["api", "enrichment-tables", "sinks", "sources", "transforms", "secrets", "vrl/stdlib"] +base = ["api", "enrichment-tables", "sinks", "sources", "transforms", "secrets", "vrl/stdlib", "codecs-parquet"] enable-api-client = ["base", "api-client"] enable-unix = ["enable-api-client", "sources-dnstap", "unix"] diff --git a/changelog.d/25313_enable_codecs_parquet_in_releases.fix.md b/changelog.d/25313_enable_codecs_parquet_in_releases.fix.md new file mode 100644 index 0000000000000..fe3baaf110d5e --- /dev/null +++ b/changelog.d/25313_enable_codecs_parquet_in_releases.fix.md @@ -0,0 +1,3 @@ +Parquet encoding in the `aws_s3` sink (`batch_encoding`) now works out of the box in the official release binaries. Previously it required compiling Vector from source with the `codecs-parquet` feature. + +authors: pront diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index 6c600b3777d18..e4b3ca565887f 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -40,12 +40,12 @@ pub struct ArrowStreamSerializerConfig { /// Allow null values for non-nullable fields in the schema. /// - /// When enabled, missing or incompatible values will be encoded as null even for fields + /// When enabled, missing or incompatible values are encoded as null, even for fields /// marked as non-nullable in the Arrow schema. This is useful when working with downstream /// systems that can handle null values through defaults, computed columns, or other mechanisms. /// - /// When disabled (default), missing values for non-nullable fields will cause encoding errors, - /// ensuring all required data is present before sending to the sink. + /// When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + /// help ensure all required data is present before sending it to the sink. #[serde(default)] #[configurable(derived)] pub allow_nullable_fields: bool, diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs index adb5b4082749b..b8874d9b1de90 100644 --- a/lib/codecs/src/encoding/format/parquet.rs +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -35,17 +35,20 @@ type EventsDroppedError = ComponentEventsDropped<'static, UNINTENTIONAL>; /// Compression algorithm and optional level for archive objects. #[configurable_component] #[derive(Default, Copy, Clone, Debug, PartialEq)] +#[configurable(metadata( + docs::enum_tag_description = "Compression codec applied per column page inside the Parquet file." +))] #[serde(tag = "algorithm", rename_all = "snake_case")] pub enum ParquetCompression { /// Zstd compression. Level must be between 1 and 21. Zstd { - /// Compression level (1–21). This is the range Vector currently supports; higher values compress more but are slower. + /// Compression level (1–21). This is the range Vector supports; higher values compress more but are slower. #[configurable(validation(range(min = 1, max = 21)))] level: u8, }, /// Gzip compression. Level must be between 1 and 9. Gzip { - /// Compression level (1–9). This is the range Vector currently supports; higher values compress more but are slower. + /// Compression level (1–9). This is the range Vector supports; higher values compress more but are slower. #[configurable(validation(range(min = 1, max = 9)))] level: u8, }, diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 61b82105a1211..6e8d2e248c518 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -111,9 +111,11 @@ pub struct S3SinkConfig { /// Batch encoding configuration for columnar formats. /// - /// When set, events are encoded together as a batch in a columnar format (e.g., Parquet) + /// When set, events are encoded together as a batch in a columnar format (for example, Parquet) /// instead of the standard per-event framing-based encoding. The columnar format handles /// its own internal compression, so the top-level `compression` setting is bypassed. + /// + /// Only the `parquet` codec is supported by the AWS S3 sink. #[cfg(feature = "codecs-parquet")] #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index d7af3413c6b97..435b2504e6a9e 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -104,6 +104,8 @@ pub struct ClickhouseConfig { /// /// When specified, events are encoded together as a single batch. /// This is mutually exclusive with per-event encoding based on the `format` field. + /// + /// Only the `arrow_stream` codec is supported by the ClickHouse sink. #[configurable(derived)] #[serde(default)] pub batch_encoding: Option, diff --git a/website/cue/reference/components/sinks/generated/aws_s3.cue b/website/cue/reference/components/sinks/generated/aws_s3.cue index 6c6dffb4fb3cb..1a4713e2897dd 100644 --- a/website/cue/reference/components/sinks/generated/aws_s3.cue +++ b/website/cue/reference/components/sinks/generated/aws_s3.cue @@ -256,6 +256,105 @@ generated: components: sinks: aws_s3: configuration: { } } } + batch_encoding: { + description: """ + Batch encoding configuration for columnar formats. + + When set, events are encoded together as a batch in a columnar format (for example, Parquet) + instead of the standard per-event framing-based encoding. The columnar format handles + its own internal compression, so the top-level `compression` setting is bypassed. + + Only the `parquet` codec is supported by the AWS S3 sink. + """ + required: false + type: object: options: { + allow_nullable_fields: { + description: """ + Allow null values for non-nullable fields in the schema. + + When enabled, missing or incompatible values are encoded as null, even for fields + marked as non-nullable in the Arrow schema. This is useful when working with downstream + systems that can handle null values through defaults, computed columns, or other mechanisms. + + When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + help ensure all required data is present before sending it to the sink. + """ + relevant_when: "codec = \"arrow_stream\"" + required: false + type: bool: default: false + } + codec: { + description: "The codec to use for batch encoding events." + required: true + type: string: enum: { + arrow_stream: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. + + [apache_arrow]: https://arrow.apache.org/ + """ + parquet: """ + Encodes events in [Apache Parquet][apache_parquet] columnar format. + + [apache_parquet]: https://parquet.apache.org/ + """ + } + } + compression: { + description: "Compression codec applied per column page inside the Parquet file." + relevant_when: "codec = \"parquet\"" + required: false + type: object: options: { + algorithm: { + description: "Compression codec applied per column page inside the Parquet file." + required: false + type: string: { + default: "snappy" + enum: { + gzip: "Gzip compression. Level must be between 1 and 9." + lz4: "LZ4 raw compression" + none: "No compression" + snappy: "Snappy compression (no level)." + zstd: "Zstd compression. Level must be between 1 and 21." + } + } + } + level: { + description: "Compression level (1–21). This is the range Vector supports; higher values compress more but are slower." + relevant_when: "algorithm = \"zstd\" or algorithm = \"gzip\"" + required: true + type: uint: {} + } + } + } + schema_file: { + description: """ + Path to a native Parquet schema file (`.schema`). + + Required unless `schema_mode` is `auto_infer`. The file must contain a valid + Parquet message type definition. + """ + relevant_when: "codec = \"parquet\"" + required: false + type: string: {} + } + schema_mode: { + description: "Controls how events with fields not present in the schema are handled." + relevant_when: "codec = \"parquet\"" + required: false + type: string: { + default: "relaxed" + enum: { + auto_infer: "Auto infer schema based on the batch. No schema file needed." + relaxed: "Missing fields become null. Extra fields are silently dropped." + strict: "Missing fields become null. Extra fields cause an error." + } + } + } + } + } bucket: { description: """ The S3 bucket name. diff --git a/website/cue/reference/components/sinks/generated/clickhouse.cue b/website/cue/reference/components/sinks/generated/clickhouse.cue index a0f4399bdc112..547fbc943e9d9 100644 --- a/website/cue/reference/components/sinks/generated/clickhouse.cue +++ b/website/cue/reference/components/sinks/generated/clickhouse.cue @@ -249,6 +249,8 @@ generated: components: sinks: clickhouse: configuration: { When specified, events are encoded together as a single batch. This is mutually exclusive with per-event encoding based on the `format` field. + + Only the `arrow_stream` codec is supported by the ClickHouse sink. """ required: false type: object: options: { @@ -256,34 +258,86 @@ generated: components: sinks: clickhouse: configuration: { description: """ Allow null values for non-nullable fields in the schema. - When enabled, missing or incompatible values will be encoded as null even for fields + When enabled, missing or incompatible values are encoded as null, even for fields marked as non-nullable in the Arrow schema. This is useful when working with downstream systems that can handle null values through defaults, computed columns, or other mechanisms. - When disabled (default), missing values for non-nullable fields will cause encoding errors, - ensuring all required data is present before sending to the sink. + When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + help ensure all required data is present before sending it to the sink. """ - required: false + relevant_when: "codec = \"arrow_stream\"" + required: false type: bool: default: false } codec: { - description: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + description: "The codec to use for batch encoding events." + required: true + type: string: enum: { + arrow_stream: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. - [apache_arrow]: https://arrow.apache.org/ - """ - required: true - type: string: enum: arrow_stream: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + [apache_arrow]: https://arrow.apache.org/ + """ + parquet: """ + Encodes events in [Apache Parquet][apache_parquet] columnar format. - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. + [apache_parquet]: https://parquet.apache.org/ + """ + } + } + compression: { + description: "Compression codec applied per column page inside the Parquet file." + relevant_when: "codec = \"parquet\"" + required: false + type: object: options: { + algorithm: { + description: "Compression codec applied per column page inside the Parquet file." + required: false + type: string: { + default: "snappy" + enum: { + gzip: "Gzip compression. Level must be between 1 and 9." + lz4: "LZ4 raw compression" + none: "No compression" + snappy: "Snappy compression (no level)." + zstd: "Zstd compression. Level must be between 1 and 21." + } + } + } + level: { + description: "Compression level (1–21). This is the range Vector supports; higher values compress more but are slower." + relevant_when: "algorithm = \"zstd\" or algorithm = \"gzip\"" + required: true + type: uint: {} + } + } + } + schema_file: { + description: """ + Path to a native Parquet schema file (`.schema`). - [apache_arrow]: https://arrow.apache.org/ + Required unless `schema_mode` is `auto_infer`. The file must contain a valid + Parquet message type definition. """ + relevant_when: "codec = \"parquet\"" + required: false + type: string: {} + } + schema_mode: { + description: "Controls how events with fields not present in the schema are handled." + relevant_when: "codec = \"parquet\"" + required: false + type: string: { + default: "relaxed" + enum: { + auto_infer: "Auto infer schema based on the batch. No schema file needed." + relaxed: "Missing fields become null. Extra fields are silently dropped." + strict: "Missing fields become null. Extra fields cause an error." + } + } } } } From 233a35c47eab1a0691b39e4af06991dfe4b0f571 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 28 Apr 2026 19:33:23 -0400 Subject: [PATCH 129/364] fix(ci): correct cross-build artifact name and path (#25282) --- .github/workflows/cross.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index bc2b2b559503b..ec0a7f33f7076 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -57,5 +57,5 @@ jobs: - run: make cross-build-${{ matrix.target }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: "vector-debug-${{ matrix.target }}" - path: "./target/${{ matrix.target }}/debug/vector" + name: "vector-${{ matrix.target }}" + path: "./target/${{ matrix.target }}/release/vector" From 8b465f6406fd088302d42f7e879297937bb8783f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2026 17:44:41 +0000 Subject: [PATCH 130/364] chore(deps): bump the patches group across 1 directory with 13 updates (#25283) * chore(deps): bump the patches group across 1 directory with 13 updates Bumps the patches group with 13 updates in the / directory: | Package | From | To | | --- | --- | --- | | [async-rs](https://github.com/amqp-rs/async-rs) | `0.8.1` | `0.8.4` | | [async-compression](https://github.com/Nullus157/async-compression) | `0.4.41` | `0.4.42` | | [inventory](https://github.com/dtolnay/inventory) | `0.3.22` | `0.3.24` | | [postgres-openssl](https://github.com/rust-postgres/rust-postgres) | `0.5.2` | `0.5.3` | | [pulsar](https://github.com/streamnative/pulsar-rs) | `6.7.1` | `6.7.2` | | [roaring](https://github.com/RoaringBitmap/roaring-rs) | `0.11.3` | `0.11.4` | | [libc](https://github.com/rust-lang/libc) | `0.2.182` | `0.2.186` | | [pastey](https://github.com/as1100k/pastey) | `0.2.1` | `0.2.2` | | [semver](https://github.com/dtolnay/semver) | `1.0.27` | `1.0.28` | | [env_logger](https://github.com/rust-cli/env_logger) | `0.11.6` | `0.11.9` | | [quote](https://github.com/dtolnay/quote) | `1.0.44` | `1.0.45` | | [schannel](https://github.com/steffengy/schannel-rs) | `0.1.28` | `0.1.29` | | [web-sys](https://github.com/wasm-bindgen/wasm-bindgen) | `0.3.91` | `0.3.97` | Updates `async-rs` from 0.8.1 to 0.8.4 - [Commits](https://github.com/amqp-rs/async-rs/compare/v0.8.1...v0.8.4) Updates `async-compression` from 0.4.41 to 0.4.42 - [Release notes](https://github.com/Nullus157/async-compression/releases) - [Commits](https://github.com/Nullus157/async-compression/compare/async-compression-v0.4.41...async-compression-v0.4.42) Updates `inventory` from 0.3.22 to 0.3.24 - [Release notes](https://github.com/dtolnay/inventory/releases) - [Commits](https://github.com/dtolnay/inventory/compare/0.3.22...0.3.24) Updates `postgres-openssl` from 0.5.2 to 0.5.3 - [Release notes](https://github.com/rust-postgres/rust-postgres/releases) - [Commits](https://github.com/rust-postgres/rust-postgres/compare/postgres-openssl-v0.5.2...postgres-openssl-v0.5.3) Updates `pulsar` from 6.7.1 to 6.7.2 - [Release notes](https://github.com/streamnative/pulsar-rs/releases) - [Commits](https://github.com/streamnative/pulsar-rs/compare/v6.7.1...v6.7.2) Updates `roaring` from 0.11.3 to 0.11.4 - [Release notes](https://github.com/RoaringBitmap/roaring-rs/releases) - [Commits](https://github.com/RoaringBitmap/roaring-rs/compare/v0.11.3...v0.11.4) Updates `libc` from 0.2.182 to 0.2.186 - [Release notes](https://github.com/rust-lang/libc/releases) - [Changelog](https://github.com/rust-lang/libc/blob/0.2.186/CHANGELOG.md) - [Commits](https://github.com/rust-lang/libc/compare/0.2.182...0.2.186) Updates `pastey` from 0.2.1 to 0.2.2 - [Release notes](https://github.com/as1100k/pastey/releases) - [Changelog](https://github.com/AS1100K/pastey/blob/master/CHANGELOG.md) - [Commits](https://github.com/as1100k/pastey/compare/v0.2.1...v0.2.2) Updates `semver` from 1.0.27 to 1.0.28 - [Release notes](https://github.com/dtolnay/semver/releases) - [Commits](https://github.com/dtolnay/semver/compare/1.0.27...1.0.28) Updates `env_logger` from 0.11.6 to 0.11.9 - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.11.6...v0.11.9) Updates `quote` from 1.0.44 to 1.0.45 - [Release notes](https://github.com/dtolnay/quote/releases) - [Commits](https://github.com/dtolnay/quote/compare/1.0.44...1.0.45) Updates `schannel` from 0.1.28 to 0.1.29 - [Release notes](https://github.com/steffengy/schannel-rs/releases) - [Commits](https://github.com/steffengy/schannel-rs/compare/v0.1.28...v0.1.29) Updates `web-sys` from 0.3.91 to 0.3.97 - [Release notes](https://github.com/wasm-bindgen/wasm-bindgen/releases) - [Changelog](https://github.com/wasm-bindgen/wasm-bindgen/blob/main/CHANGELOG.md) - [Commits](https://github.com/wasm-bindgen/wasm-bindgen/commits) --- updated-dependencies: - dependency-name: async-compression dependency-version: 0.4.42 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: async-rs dependency-version: 0.8.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: env_logger dependency-version: 0.11.9 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: inventory dependency-version: 0.3.24 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: libc dependency-version: 0.2.186 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: pastey dependency-version: 0.2.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: postgres-openssl dependency-version: 0.5.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: pulsar dependency-version: 6.7.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: quote dependency-version: 1.0.45 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: roaring dependency-version: 0.11.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: schannel dependency-version: 0.1.29 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: semver dependency-version: 1.0.28 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: web-sys dependency-version: 0.3.95 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches ... Signed-off-by: dependabot[bot] * update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 433 +++++++++++++++++++----------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 7 + lib/file-source-common/Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 6 files changed, 293 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ad2d1635d855..52ed5cd06c593 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,7 +327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6368f9ae5c6ec403ca910327ae0c9437b0a85255b6950c90d497e6177f6e5e" dependencies = [ "proc-macro-hack", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -847,9 +847,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -1019,15 +1019,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] [[package]] name = "async-rs" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d47e5e0df9bce1f132f67a64409a973b031481366b858330013060ca76b3a80" +checksum = "31ebdd144e4199c67b5134fe2280e40d9656071f11df525d3322b43b6534c714" dependencies = [ "async-compat", "async-global-executor", @@ -1035,7 +1035,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "hickory-resolver 0.25.2", + "hickory-resolver 0.26.0", "tokio", "tokio-stream", ] @@ -1076,7 +1076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -1093,7 +1093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -1940,7 +1940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94f7cc1bbae04cfe11de9e39e2c6dc755947901e0f4e76180ab542b6deb5e15e" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "tracing 0.1.44", ] @@ -2069,7 +2069,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "regex", "rustc-hash", "shlex", @@ -2112,7 +2112,7 @@ version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6cbbb8f56245b5a479b30a62cdc86d26e2f35c2b9f594bc4671654b03851380" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -2238,7 +2238,7 @@ dependencies = [ "ident_case", "prettyplease", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustversion", "syn 2.0.117", ] @@ -2268,7 +2268,7 @@ dependencies = [ "once_cell", "proc-macro-crate 3.2.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -2362,7 +2362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -2667,7 +2667,7 @@ checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -2841,9 +2841,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "brotli", "compression-core", @@ -2855,9 +2855,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -3352,7 +3352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3385,7 +3385,7 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "strsim", "syn 2.0.117", ] @@ -3399,7 +3399,7 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "strsim", "syn 2.0.117", ] @@ -3411,7 +3411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3422,7 +3422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3555,7 +3555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -3566,7 +3566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3577,7 +3577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3588,7 +3588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3609,7 +3609,7 @@ checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3631,7 +3631,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustc_version", "syn 1.0.109", ] @@ -3653,7 +3653,7 @@ checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case 0.10.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustc_version", "syn 2.0.117", "unicode-xid 0.2.4", @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3832,7 +3832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3907,7 +3907,7 @@ checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" dependencies = [ "enum-ordinalize", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4009,7 +4009,7 @@ checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4029,7 +4029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4041,7 +4041,7 @@ checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" dependencies = [ "once_cell", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4061,7 +4061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4073,9 +4073,9 @@ checksum = "62a61b2faff777e62dbccd7f82541d873f96264d050c5dd7e95194f79fc4de29" [[package]] name = "env_filter" -version = "0.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -4083,14 +4083,14 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.6" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" dependencies = [ "anstream", "anstyle", "env_filter", - "humantime", + "jiff", "log", ] @@ -4190,7 +4190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "332b1937705b7ed2fce76837024e9ae6f41cd2ad18a32c052de081f89982561b" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -4530,7 +4530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -5088,6 +5088,30 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61c8db47fae51ba9f8f2a2748bd87542acfbe22f2ec9cf9c8ec72d1ee6e9a6" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto 0.26.0", + "idna", + "ipnet", + "jni 0.22.4", + "rand 0.10.1", + "thiserror 2.0.17", + "tinyvec", + "tokio", + "tracing 0.1.44", + "url", +] + [[package]] name = "hickory-proto" version = "0.24.4" @@ -5135,7 +5159,26 @@ dependencies = [ "thiserror 2.0.17", "time", "tinyvec", - "tokio", + "tracing 0.1.44", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a916d0494600d99ecb15aadfab677ad97c4de559e8f1af0c129353a733ac1fcc" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni 0.22.4", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.17", + "tinyvec", "tracing 0.1.44", "url", ] @@ -5163,20 +5206,25 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "a10bd64d950b4d38ca21e25c8ae230712e4955fb8290cfcb29a5e5dc6017e544" dependencies = [ "cfg-if", "futures-util", - "hickory-proto 0.25.2", + "hickory-net", + "hickory-proto 0.26.0", "ipconfig", + "ipnet", + "jni 0.22.4", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.4", + "rand 0.10.1", "resolv-conf", "smallvec", + "system-configuration 0.7.0", "thiserror 2.0.17", "tokio", "tracing 0.1.44", @@ -5703,7 +5751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -5835,7 +5883,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -5847,9 +5895,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -5988,7 +6036,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6001,19 +6049,68 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.0", "log", "thiserror 1.0.68", "walkdir", "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.17", + "walkdir", + "windows-link 0.2.0", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.45", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + [[package]] name = "jni-sys" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote 1.0.45", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.32" @@ -6025,10 +6122,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -6398,9 +6497,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libflate" @@ -6695,7 +6794,7 @@ checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" dependencies = [ "macro_magic_core", "macro_magic_macros", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6709,7 +6808,7 @@ dependencies = [ "derive-syn-parse", "macro_magic_core_macros", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6720,7 +6819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6731,7 +6830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6916,7 +7015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6988,7 +7087,7 @@ dependencies = [ "once_cell", "proc-macro-error2", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "regex", "syn 2.0.117", ] @@ -7099,7 +7198,7 @@ checksum = "63981427a0f26b89632fd2574280e069d09fb2912a3138da15de0174d11dd077" dependencies = [ "macro_magic", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7515,7 +7614,7 @@ checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7527,7 +7626,7 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7731,7 +7830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7948,9 +8047,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pastey" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec" +checksum = "c5a797f0e07bdf071d15742978fc3128ec6c22891c31a3a931513263904c982a" [[package]] name = "pbkdf2" @@ -8021,7 +8120,7 @@ dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8107,7 +8206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8238,9 +8337,9 @@ dependencies = [ [[package]] name = "postgres-openssl" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f86f073ad570f76e9e278ce6f05775fc723eed7daa6b4f9c2aa078080a564a0" +checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4" dependencies = [ "openssl", "tokio", @@ -8324,6 +8423,17 @@ dependencies = [ "termtree", ] +[[package]] +name = "prefix-trie" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23370be78b7e5bcbb0cab4a02047eb040279a693c78daad04c2c5f1c24a83503" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettydiff" version = "0.8.0" @@ -8392,7 +8502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", ] [[package]] @@ -8403,7 +8513,7 @@ checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" dependencies = [ "proc-macro-error-attr2", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8497,7 +8607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8581,7 +8691,7 @@ dependencies = [ "anyhow", "itertools 0.12.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8594,7 +8704,7 @@ dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8607,7 +8717,7 @@ dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8683,7 +8793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -8699,9 +8809,9 @@ dependencies = [ [[package]] name = "pulsar" -version = "6.7.1" +version = "6.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c00856d6c64af2078a74c2de029842cdaece2f35b1223f5166809ea49cf5cd" +checksum = "e2367cb38f1b65857bc11dd13b2adf13b7a1d991ef1cd43572f1420958c56cc2" dependencies = [ "async-channel", "async-trait", @@ -8825,7 +8935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9a28b8493dd664c8b171dd944da82d933f7d456b829bfb236738e1fe06c5ba4" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8893,9 +9003,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2 1.0.106", ] @@ -9261,7 +9371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -9550,7 +9660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -9592,9 +9702,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ "bytemuck", "byteorder", @@ -9650,7 +9760,7 @@ dependencies = [ "glob", "proc-macro-crate 3.2.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "regex", "relative-path 1.9.3", "rustc_version", @@ -9955,9 +10065,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.0", ] @@ -10081,9 +10191,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -10162,7 +10272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10173,7 +10283,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10226,7 +10336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10279,7 +10389,7 @@ checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10331,7 +10441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10461,6 +10571,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -10600,7 +10720,7 @@ checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -10612,7 +10732,7 @@ checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10624,7 +10744,7 @@ checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10751,7 +10871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "sqlx-core", "sqlx-macros-core", "syn 2.0.117", @@ -10769,7 +10889,7 @@ dependencies = [ "hex", "once_cell", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "serde", "serde_json", "sha2", @@ -10988,7 +11108,7 @@ checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustversion", "syn 2.0.117", ] @@ -11001,7 +11121,7 @@ checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustversion", "syn 2.0.117", ] @@ -11014,7 +11134,7 @@ checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11026,7 +11146,7 @@ checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11073,7 +11193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "unicode-ident", ] @@ -11084,7 +11204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "unicode-ident", ] @@ -11110,7 +11230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11183,6 +11303,17 @@ dependencies = [ "system-configuration-sys 0.6.0", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.3", + "system-configuration-sys 0.6.0", +] + [[package]] name = "system-configuration-sys" version = "0.5.0" @@ -11337,7 +11468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11348,7 +11479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11505,7 +11636,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11859,7 +11990,7 @@ dependencies = [ "prettyplease", "proc-macro2 1.0.106", "prost-build 0.12.6", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12042,7 +12173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12168,7 +12299,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12255,7 +12386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12266,7 +12397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12323,7 +12454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ecee5b05c459ea4cd97df7685db58699c32e070465f88dc806d7c98a5088edc" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustc_version", "syn 2.0.117", ] @@ -12348,7 +12479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12912,7 +13043,7 @@ name = "vector-common-macros" version = "0.1.0" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12948,7 +13079,7 @@ dependencies = [ "convert_case 0.8.0", "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "serde", "serde_json", "syn 2.0.117", @@ -12961,7 +13092,7 @@ version = "0.1.0" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "serde", "serde_derive_internals", "syn 2.0.117", @@ -13463,9 +13594,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" dependencies = [ "cfg-if", "once_cell", @@ -13488,32 +13619,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" dependencies = [ "bumpalo", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" dependencies = [ "unicode-ident", ] @@ -13581,9 +13712,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" dependencies = [ "js-sys", "wasm-bindgen", @@ -13607,7 +13738,7 @@ checksum = "60b6f804e41d0852e16d2eaee61c7e4f7d3e8ffdb7b8ed85886aeb0791fe9fcd" dependencies = [ "core-foundation 0.9.3", "home", - "jni", + "jni 0.21.1", "log", "ndk-context", "objc", @@ -13786,7 +13917,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -13797,7 +13928,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -13808,7 +13939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -13819,7 +13950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -14320,7 +14451,7 @@ dependencies = [ "anyhow", "prettyplease", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", @@ -14425,7 +14556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "synstructure", ] @@ -14455,7 +14586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -14466,7 +14597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -14486,7 +14617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "synstructure", ] @@ -14515,7 +14646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] diff --git a/Cargo.toml b/Cargo.toml index 52161db4b2905..db582e2930491 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,7 +187,7 @@ rdkafka = { version = "0.39.0", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.40.0", default-features = false, features = ["std"] } -semver = { version = "1.0.27", default-features = false, features = ["serde", "std"] } +semver = { version = "1.0.28", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } serde_yaml = { version = "0.9.34", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ea2ff8e85d12e..31c7d69405a20 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -336,6 +336,7 @@ heim-net,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi +hickory-net,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hickory-proto,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hickory-resolver,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hkdf,https://github.com/RustCrypto/KDFs,MIT OR Apache-2.0,RustCrypto Developers @@ -398,7 +399,11 @@ itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay jiff-static,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase +jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,jni team +jni-macros,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,The jni-macros Authors +jni-sys,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,"Steven Fackler , Robert Bragg " jni-sys,https://github.com/sfackler/rust-jni-sys,MIT OR Apache-2.0,Steven Fackler +jni-sys-macros,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,Robert Bragg js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers json-patch,https://github.com/idubrov/json-patch,MIT OR Apache-2.0,Ivan Dubrov jsonpath-rust,https://github.com/besok/jsonpath-rust,MIT,BorisZhguchev @@ -568,6 +573,7 @@ postgres-protocol,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2 postgres-types,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler powerfmt,https://github.com/jhpratt/powerfmt,MIT OR Apache-2.0,Jacob Pratt ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors +prefix-trie,https://github.com/tiborschneider/prefix-trie,MIT OR Apache-2.0,The prefix-trie Authors prettydiff,https://github.com/romankoblov/prettydiff,MIT,Roman Koblov prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay prettytable-rs,https://github.com/phsym/prettytable-rs,BSD-3-Clause,Pierre-Henri Symoneaux @@ -709,6 +715,7 @@ signal-hook-registry,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Mi signatory,https://github.com/iqlusioninc/crates/tree/main/signatory,Apache-2.0 OR MIT,Tony Arcieri signature,https://github.com/RustCrypto/traits/tree/master/signature,Apache-2.0 OR MIT,RustCrypto Developers simd-adler32,https://github.com/mcountryman/simd-adler32,MIT,Marvin Countryman +simd_cesu8,https://github.com/seancroach/simd_cesu8,Apache-2.0 OR MIT,Sean C. Roach simdutf8,https://github.com/rusticstuff/simdutf8,MIT OR Apache-2.0,Hans Kratz simpl,https://github.com/durch/simplerr,MIT,Drazen Urch siphasher,https://github.com/jedisct1/rust-siphash,MIT OR Apache-2.0,Frank Denis diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml index e115f3a74bf52..6d3a885bc9a7a 100644 --- a/lib/file-source-common/Cargo.toml +++ b/lib/file-source-common/Cargo.toml @@ -20,7 +20,7 @@ serde_json = { version = "1.0.143", default-features = false } bstr = { version = "1.12", default-features = false } bytes = { version = "1.11.1", default-features = false, features = ["serde"] } dashmap = { version = "6.1", default-features = false } -async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.42", features = ["tokio", "gzip"] } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } tokio = { workspace = true, features = ["full"] } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 7e8e6b6cd1153..a6b6e94c3b27f 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -21,7 +21,7 @@ futures = { version = "0.3.31", default-features = false, features = ["executor" futures-util.workspace = true vector-common = { path = "../vector-common", default-features = false } file-source-common = { path = "../file-source-common" } -async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.42", features = ["tokio", "gzip"] } [dev-dependencies] tokio = { workspace = true, features = ["full"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index c26953cb3590a..8836c2f70beaf 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -71,7 +71,7 @@ quickcheck = { workspace = true, optional = true } security-framework = "3.6.0" [target.'cfg(windows)'.dependencies] -schannel = "0.1.28" +schannel = "0.1.29" [build-dependencies] prost-build.workspace = true From 75b9d07a8231e7c321652e4ff031d8ce0757d9ab Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 29 Apr 2026 15:57:15 -0400 Subject: [PATCH 131/364] feat(website): improve search ranking for component reference pages (#25327) * feat(website): improve search ranking for component reference pages * Improve regex match --- website/scripts/typesense-index.ts | 20 ++++++++++++++++---- website/typesense.config.json | 30 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/website/scripts/typesense-index.ts b/website/scripts/typesense-index.ts index f800e710d1754..81c241b331276 100644 --- a/website/scripts/typesense-index.ts +++ b/website/scripts/typesense-index.ts @@ -33,6 +33,8 @@ type AlgoliaRecord = { ranking: number; section: string; content: string; + componentWithType: string; + component: string; }; type Section = { @@ -94,6 +96,11 @@ async function indexHTMLFiles( const pageTitle = $("meta[name='algolia:title']").attr("content") || ""; const pageTagsString = $("meta[name='keywords']").attr("content") || ""; const pageTags: string[] = pageTagsString === "" ? [] : pageTagsString.split(","); + const pageUrl = getPageUrl(file); + const componentMatch = pageUrl.match(/\/docs\/reference\/configuration\/(sources|sinks|transforms)\/([^/]+)/); + const componentKindMap: Record = { sources: "source", sinks: "sink", transforms: "transform" }; + const pageComponent = componentMatch ? componentMatch[2] : ""; + const pageComponentWithType = componentMatch ? `${componentMatch[2]} ${componentKindMap[componentMatch[1]]}` : ""; // @ts-ignore $(".algolia-no-index").each((_, d) => $(d).remove()); @@ -130,7 +137,6 @@ async function indexHTMLFiles( let activeRecord: AlgoliaRecord | null = null; for (const item of payload) { - const pageUrl = getPageUrl(file); const itemUrl = getItemUrl(file, item); const hashedId = hashString(itemUrl); @@ -146,7 +152,9 @@ async function indexHTMLFiles( ranking, hierarchy: [], tags: pageTags, - content: "" + content: "", + componentWithType: pageComponentWithType, + component: pageComponent }; } else if (item.level === 1) { // h1 logic @@ -165,7 +173,9 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy, activeRecord.title.trim()], tags: pageTags, - content: "" + content: "", + componentWithType: pageComponentWithType, + component: pageComponent }; } else { // h2-h6 logic @@ -186,7 +196,9 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy.slice(0, lastIndex)], tags: pageTags, - content: "" + content: "", + componentWithType: pageComponentWithType, + component: pageComponent }; } diff --git a/website/typesense.config.json b/website/typesense.config.json index f0543b9133c8b..5e09a5ee14694 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -136,6 +136,32 @@ "type": "string[]", "stem_dictionary": "", "store": true + }, + { + "facet": false, + "index": true, + "infix": false, + "locale": "", + "name": "componentWithType", + "optional": true, + "sort": false, + "stem": false, + "type": "string", + "stem_dictionary": "", + "store": true + }, + { + "facet": false, + "index": true, + "infix": false, + "locale": "", + "name": "component", + "optional": true, + "sort": false, + "stem": false, + "type": "string", + "stem_dictionary": "", + "store": true } ] }, @@ -144,8 +170,8 @@ "name": "vector_docs_search", "search_parameters": { "sort_by": "_text_match:desc,ranking:desc,level:desc", - "query_by": "pageTitle,title,hierarchy,content,tags", - "query_by_weights": "4,3,2,1,2", + "query_by": "componentWithType,component,pageTitle,title,hierarchy,content,tags", + "query_by_weights": "5,4,4,3,2,1,2", "text_match_type": "sum_score" } } From 625c1d3a57a59784da7b1e42b2318d4c372649d8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 29 Apr 2026 16:06:31 -0400 Subject: [PATCH 132/364] chore(deps): bump the serde group across 1 directory with 2 updates (#25227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump the serde group across 1 directory with 2 updates Bumps the serde group with 2 updates in the / directory: [serde_with](https://github.com/jonasbb/serde_with) and [serde_json](https://github.com/serde-rs/json). Updates `serde_with` from 3.14.0 to 3.17.0 - [Release notes](https://github.com/jonasbb/serde_with/releases) - [Commits](https://github.com/jonasbb/serde_with/compare/v3.14.0...v3.17.0) Updates `serde_json` from 1.0.145 to 1.0.149 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.145...v1.0.149) --- updated-dependencies: - dependency-name: serde_with dependency-version: 3.17.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: serde - dependency-name: serde_json dependency-version: 1.0.149 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: serde ... Signed-off-by: dependabot[bot] * fix(core): update float size estimator to match serde_json's new formatter serde_json 1.0.147 switched from Ryū to Żmij for float-to-string formatting. The EstimatedJsonEncodedSizeOf impls for f32/f64 were using ryu directly to predict serde_json's output size, which silently drifts after the bump (for example ryu emits `1e16` while serde_json now emits `1e+16`). Swap to zmij so the estimate tracks what serde_json actually produces. * chore(codecs): update native_json fixtures for new serde_json float formatter Round-tripped the affected fixtures through serde_json 1.0.149 so positive exponents render as `e+N` (e.g. `1.797...e+308`) to match the new Żmij formatter introduced in 1.0.147. Event content is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): regenerate 3rd-party license file for zmij Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): allow `mij` in spell check The changelog mentions the Żmij formatter; check-spelling tokenizes around `Ż` and flags the residual `mij`. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): centralize serde_with as a workspace dependency Move the serde_with version + features into [workspace.dependencies], mirroring how serde and serde_json are already declared. The workspace entry carries the union of features previously enabled across the three dependents (std, macros, chrono_0_4); since Cargo unions features across the workspace anyway, the resulting workspace build is identical. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): use workspace serde_json in file-source-common Addresses review feedback from @thomasqueirozb on #25227. Cargo unions features across the workspace, so the union for any binary that pulls file-source-common alongside other serde_json dependents is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) * Update Cargo.lock --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Thomas --- .github/actions/spelling/expect.txt | 1 + Cargo.lock | 61 +++++++++++++++---- Cargo.toml | 3 +- LICENSE-3rdparty.csv | 1 + changelog.d/serde_group_bump.enhancement.md | 3 + lib/codecs/Cargo.toml | 2 +- .../tests/data/native_encoding/json/0605.json | 2 +- .../tests/data/native_encoding/json/0638.json | 2 +- .../tests/data/native_encoding/json/0639.json | 2 +- .../tests/data/native_encoding/json/0647.json | 2 +- .../tests/data/native_encoding/json/0700.json | 2 +- .../tests/data/native_encoding/json/0788.json | 2 +- lib/file-source-common/Cargo.toml | 2 +- lib/vector-config/Cargo.toml | 4 +- lib/vector-core/Cargo.toml | 4 +- .../event/estimated_json_encoded_size_of.rs | 4 +- 16 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 changelog.d/serde_group_bump.enhancement.md diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 3656732c51bba..353de8bdf6f91 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -350,6 +350,7 @@ mezmo mfoo mgmt MIDP +mij mingrammer mkto mlua diff --git a/Cargo.lock b/Cargo.lock index 52ed5cd06c593..9bab182e05373 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3376,6 +3376,16 @@ dependencies = [ "darling_macro 0.21.3", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + [[package]] name = "darling_core" version = "0.20.11" @@ -3404,6 +3414,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2 1.0.106", + "quote 1.0.45", + "strsim", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.20.11" @@ -3426,6 +3449,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote 1.0.45", + "syn 2.0.117", +] + [[package]] name = "dary_heap" version = "0.3.8" @@ -10289,16 +10323,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "indexmap 2.12.0", "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -10363,9 +10397,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64 0.22.1", "chrono", @@ -10374,8 +10408,7 @@ dependencies = [ "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.3", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -10383,11 +10416,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "proc-macro2 1.0.106", "quote 1.0.45", "syn 2.0.117", @@ -13148,7 +13181,6 @@ dependencies = [ "rand 0.9.4", "rand_distr", "regex", - "ryu", "schannel", "security-framework 3.6.0", "serde", @@ -13177,6 +13209,7 @@ dependencies = [ "vector-config-common", "vector-lookup", "vrl", + "zmij", ] [[package]] @@ -14656,6 +14689,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index db582e2930491..904f540cd26a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,8 @@ reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.40.0", default-features = false, features = ["std"] } semver = { version = "1.0.28", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } -serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } +serde_json = { version = "1.0.149", default-features = false, features = ["preserve_order", "raw_value", "std"] } +serde_with = { version = "3.18.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } serde_yaml = { version = "0.9.34", default-features = false } snafu = { version = "0.9.0", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 31c7d69405a20..311b35452684c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -937,6 +937,7 @@ zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MI zerovec,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers zerovec-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zlib-rs,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The zlib-rs Authors +zmij,https://github.com/dtolnay/zmij,MIT,David Tolnay zstd,https://github.com/gyscos/zstd-rs,MIT,Alexandre Bury zstd-safe,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury zstd-sys,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury diff --git a/changelog.d/serde_group_bump.enhancement.md b/changelog.d/serde_group_bump.enhancement.md new file mode 100644 index 0000000000000..3c9ea910c6601 --- /dev/null +++ b/changelog.d/serde_group_bump.enhancement.md @@ -0,0 +1,3 @@ +Bumped `serde_json` to `1.0.149` and `serde_with` to `3.18.0`. `serde_json` switched its float-to-string formatter from Ryū to Żmij in `1.0.147`, so floats serialized via the `native_json` codec may render with slightly different textual form (for example `1e+16` instead of `1e16`). The change is purely cosmetic: parsed `f32`/`f64` values round-trip identically, and Vector-to-Vector communication between old and new versions is unaffected. + +authors: pront diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index b9043b7223d8a..6a5f8424a61fd 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -45,7 +45,7 @@ prost-reflect.workspace = true rand.workspace = true regex.workspace = true serde.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } +serde_with.workspace = true serde_json.workspace = true serde-aux = { version = "4.5", optional = true } smallvec = { version = "1", default-features = false, features = ["union"] } diff --git a/lib/codecs/tests/data/native_encoding/json/0605.json b/lib/codecs/tests/data/native_encoding/json/0605.json index bcd39f82ae418..1c46a13cf2fb7 100644 --- a/lib/codecs/tests/data/native_encoding/json/0605.json +++ b/lib/codecs/tests/data/native_encoding/json/0605.json @@ -1 +1 @@ -{"metric":{"name":"l","tags":{"a":"l","e":"l","q":"n"},"timestamp":"1970-01-01T07:31:02.000011250Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-209216.0,"avg":-113920.0}}}}} \ No newline at end of file +{"metric":{"name":"l","tags":{"a":"l","e":"l","q":"n"},"timestamp":"1970-01-01T07:31:02.000011250Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-209216.0,"avg":-113920.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0638.json b/lib/codecs/tests/data/native_encoding/json/0638.json index 61f4ec83d0fb4..85fb25087875f 100644 --- a/lib/codecs/tests/data/native_encoding/json/0638.json +++ b/lib/codecs/tests/data/native_encoding/json/0638.json @@ -1 +1 @@ -{"metric":{"name":"t","namespace":"k","tags":{"s":"b","x":"y"},"timestamp":"1969-12-31T19:41:49.000002928Z","interval_ms":2984613283,"kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-667968.0,"avg":-404160.0}}}}} \ No newline at end of file +{"metric":{"name":"t","namespace":"k","tags":{"s":"b","x":"y"},"timestamp":"1969-12-31T19:41:49.000002928Z","interval_ms":2984613283,"kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-667968.0,"avg":-404160.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0639.json b/lib/codecs/tests/data/native_encoding/json/0639.json index 3f8fd591f1128..9bbe86b1aa695 100644 --- a/lib/codecs/tests/data/native_encoding/json/0639.json +++ b/lib/codecs/tests/data/native_encoding/json/0639.json @@ -1 +1 @@ -{"metric":{"name":"y","namespace":"d","kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-422464.0,"avg":906752.0}}}}} \ No newline at end of file +{"metric":{"name":"y","namespace":"d","kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-422464.0,"avg":906752.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0647.json b/lib/codecs/tests/data/native_encoding/json/0647.json index 8dbe86fb16fdd..c6e6cdb1dfbab 100644 --- a/lib/codecs/tests/data/native_encoding/json/0647.json +++ b/lib/codecs/tests/data/native_encoding/json/0647.json @@ -1 +1 @@ -{"metric":{"name":"b","interval_ms":586459638,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":999808.0,"avg":-148288.0}}}}} \ No newline at end of file +{"metric":{"name":"b","interval_ms":586459638,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":999808.0,"avg":-148288.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0700.json b/lib/codecs/tests/data/native_encoding/json/0700.json index 42fdfec5980f1..da5055a337409 100644 --- a/lib/codecs/tests/data/native_encoding/json/0700.json +++ b/lib/codecs/tests/data/native_encoding/json/0700.json @@ -1 +1 @@ -{"metric":{"name":"j","tags":{"f":"k"},"timestamp":"1969-12-31T18:05:17.000026769Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-481728.0,"avg":590528.0}}}}} \ No newline at end of file +{"metric":{"name":"j","tags":{"f":"k"},"timestamp":"1969-12-31T18:05:17.000026769Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-481728.0,"avg":590528.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0788.json b/lib/codecs/tests/data/native_encoding/json/0788.json index f458d97be6718..13f2d0913ec85 100644 --- a/lib/codecs/tests/data/native_encoding/json/0788.json +++ b/lib/codecs/tests/data/native_encoding/json/0788.json @@ -1 +1 @@ -{"metric":{"name":"w","interval_ms":870813348,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-356992.0,"avg":-572800.0}}}}} \ No newline at end of file +{"metric":{"name":"w","interval_ms":870813348,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-356992.0,"avg":-572800.0}}}}} \ No newline at end of file diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml index 6d3a885bc9a7a..f7a191d9cae7d 100644 --- a/lib/file-source-common/Cargo.toml +++ b/lib/file-source-common/Cargo.toml @@ -16,7 +16,7 @@ chrono.workspace = true tracing.workspace = true crc = "3.3.0" serde = { version = "1.0", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.143", default-features = false } +serde_json.workspace = true bstr = { version = "1.12", default-features = false } bytes = { version = "1.11.1", default-features = false, features = ["serde"] } dashmap = { version = "6.1", default-features = false } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 2283df3d133e1..5f4cfdbdcf1b2 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -20,7 +20,7 @@ no-proxy = { version = "0.3.6", default-features = false, features = ["serialize num-traits = { version = "0.2.19", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std"] } +serde_with.workspace = true snafu.workspace = true toml.workspace = true tracing.workspace = true @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } +serde_with.workspace = true diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 8836c2f70beaf..5c3370ac1a526 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -42,10 +42,10 @@ prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } -ryu = { version = "1", default-features = false } +zmij = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } +serde_with.workspace = true smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true socket2.workspace = true diff --git a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs index 832dc70cfd343..c6e8c17bc95b2 100644 --- a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs +++ b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs @@ -120,13 +120,13 @@ impl EstimatedJsonEncodedSizeOf for bool { impl EstimatedJsonEncodedSizeOf for f64 { fn estimated_json_encoded_size_of(&self) -> JsonSize { - ryu::Buffer::new().format_finite(*self).len().into() + zmij::Buffer::new().format_finite(*self).len().into() } } impl EstimatedJsonEncodedSizeOf for f32 { fn estimated_json_encoded_size_of(&self) -> JsonSize { - ryu::Buffer::new().format_finite(*self).len().into() + zmij::Buffer::new().format_finite(*self).len().into() } } From b5fb618eb24bade72d5277be6d23836810d24fa8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 30 Apr 2026 17:53:00 -0400 Subject: [PATCH 133/364] chore(dev): remove unused update_counter macro (#25333) --- lib/vector-core/src/metrics/mod.rs | 44 ------------------------------ 1 file changed, 44 deletions(-) diff --git a/lib/vector-core/src/metrics/mod.rs b/lib/vector-core/src/metrics/mod.rs index 660f29c000db3..cd900d392f004 100644 --- a/lib/vector-core/src/metrics/mod.rs +++ b/lib/vector-core/src/metrics/mod.rs @@ -200,50 +200,6 @@ impl Controller { } } -#[macro_export] -/// This macro is used to emit metrics as a `counter` while simultaneously -/// converting from absolute values to incremental values. -/// -/// Values that do not arrive in strictly monotonically increasing order are -/// ignored and will not be emitted. -macro_rules! update_counter { - ($label:literal, $value:expr) => {{ - use ::std::sync::atomic::{AtomicU64, Ordering}; - - static PREVIOUS_VALUE: AtomicU64 = AtomicU64::new(0); - - let new_value = $value; - let mut previous_value = PREVIOUS_VALUE.load(Ordering::Relaxed); - - loop { - // Either a new greater value has been emitted before this thread updated the counter - // or values were provided that are not in strictly monotonically increasing order. - // Ignore. - if new_value <= previous_value { - break; - } - - match PREVIOUS_VALUE.compare_exchange_weak( - previous_value, - new_value, - Ordering::SeqCst, - Ordering::Relaxed, - ) { - // Another thread has written a new value before us. Re-enter loop. - Err(value) => previous_value = value, - // Calculate delta to last emitted value and emit it. - Ok(_) => { - let delta = new_value - previous_value; - // Albeit very unlikely, note that this sequence of deltas might be emitted in - // a different order than they were calculated. - counter!($label).increment(delta); - break; - } - } - } - }}; -} - #[cfg(test)] mod tests { use super::*; From bd8ab1a245a439134b2bd9f2cb540ca550db860c Mon Sep 17 00:00:00 2001 From: Andy <26137827+ndrsg@users.noreply.github.com> Date: Fri, 1 May 2026 15:30:24 +0200 Subject: [PATCH 134/364] feat(sinks): introduce RetryStrategy / Config for http based sinks (#25057) * feat(sinks): introduce a configurable retry strategy for http * feat(sinks): wire the RetryStrategy with the HttpRetryLogic * feat(sinks): wire RetryStrategy with all HttpStatusRetryLogic and all affected sinks * docs(sinks): http example config with custom retry strategy * feat(http sink): add integration test for custom retry strategy * docs(sinks): add changelog fragment * fix(http sink): update is_retriable_error logic to respect retry strategy * fix(http sink): add handling for NOT_IMPLEMENTED status in retry strategy * chore(sinks): format code * feat(sinks): implement timeout retry logic and add tests for non-retriable timeouts * docs(sinks): clarify that success status codes are not retried on RetryAll strategy * feat(sinks): improve typesafety of RetryStrategy StatusCode * docs(sinks http): enhance transport error classification for HTTP sinks with shared retry logic * feat(sinks http): add debug logging for unsuccessful HTTP responses in retry logic * refactor(http sink): avoid heap allocation on reasoning happy path * fix(http sink): format example config * chore: add custom attributes for numeric type in StatusCode metadata * docs(sinks http): update docs for http sinks supporting retry_strategy --------- Co-authored-by: Pavlos Rontidis --- benches/http.rs | 1 + .../10870_http_retry_strategy.feature.md | 7 + ...http_retry_transport_errors.enhancement.md | 9 + config/examples/http_sink_custom_retry.yaml | 42 +++ lib/vector-config/src/http.rs | 6 + src/sinks/appsignal/config.rs | 11 +- src/sinks/axiom/config.rs | 8 +- src/sinks/azure_logs_ingestion/config.rs | 16 +- src/sinks/azure_monitor_logs/config.rs | 16 +- src/sinks/datadog/events/config.rs | 14 +- src/sinks/gcp/stackdriver/logs/config.rs | 11 +- src/sinks/gcp/stackdriver/metrics/config.rs | 12 +- src/sinks/honeycomb/config.rs | 11 +- src/sinks/http/config.rs | 15 +- src/sinks/http/tests.rs | 100 +++++++ src/sinks/keep/config.rs | 11 +- src/sinks/opentelemetry/mod.rs | 1 + src/sinks/prometheus/remote_write/config.rs | 11 +- src/sinks/util/http.rs | 282 ++++++++++++++++-- src/sinks/util/retries.rs | 60 +++- .../components/sinks/generated/appsignal.cue | 31 ++ .../components/sinks/generated/axiom.cue | 31 ++ .../sinks/generated/azure_logs_ingestion.cue | 31 ++ .../sinks/generated/azure_monitor_logs.cue | 31 ++ .../sinks/generated/datadog_events.cue | 31 ++ .../sinks/generated/gcp_stackdriver_logs.cue | 31 ++ .../generated/gcp_stackdriver_metrics.cue | 31 ++ .../components/sinks/generated/honeycomb.cue | 31 ++ .../components/sinks/generated/http.cue | 31 ++ .../components/sinks/generated/keep.cue | 31 ++ .../sinks/generated/opentelemetry.cue | 31 ++ .../generated/prometheus_remote_write.cue | 31 ++ 32 files changed, 957 insertions(+), 59 deletions(-) create mode 100644 changelog.d/10870_http_retry_strategy.feature.md create mode 100644 changelog.d/10870_http_retry_transport_errors.enhancement.md create mode 100644 config/examples/http_sink_custom_retry.yaml diff --git a/benches/http.rs b/benches/http.rs index 30b35b8b257f7..5c8b5eaac7efe 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -63,6 +63,7 @@ fn benchmark_http(c: &mut Criterion) { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), }, ); diff --git a/changelog.d/10870_http_retry_strategy.feature.md b/changelog.d/10870_http_retry_strategy.feature.md new file mode 100644 index 0000000000000..09afba2eddd40 --- /dev/null +++ b/changelog.d/10870_http_retry_strategy.feature.md @@ -0,0 +1,7 @@ +HTTP-based sinks that use the shared retry helpers now support a `retry_strategy` configuration +option to control which HTTP response codes are retried. The `http` sink also includes a new +example showing how to retry only specific transient status codes. + +Issue: https://github.com/vectordotdev/vector/issues/10870 + +authors: ndrsg diff --git a/changelog.d/10870_http_retry_transport_errors.enhancement.md b/changelog.d/10870_http_retry_transport_errors.enhancement.md new file mode 100644 index 0000000000000..f06f63b2f0e07 --- /dev/null +++ b/changelog.d/10870_http_retry_transport_errors.enhancement.md @@ -0,0 +1,9 @@ +HTTP-based sinks using the shared retry logic now classify transport-layer failures with +`HttpError::is_retriable`: connection and TLS connector issues may be retried, while failures +such as invalid HTTP request construction or an invalid proxy URI are not. Setting +`retry_strategy` to `none` disables retries for these transport errors and for request +timeouts, in addition to status-code-based retries. + +Issue: https://github.com/vectordotdev/vector/issues/10870 + +authors: ndrsg diff --git a/config/examples/http_sink_custom_retry.yaml b/config/examples/http_sink_custom_retry.yaml new file mode 100644 index 0000000000000..fd973c890e991 --- /dev/null +++ b/config/examples/http_sink_custom_retry.yaml @@ -0,0 +1,42 @@ +# HTTP sink example with a custom retry strategy +# ---------------------------------------------------- +# Sends demo logs to an HTTP endpoint and only retries the response codes +# that the upstream API documents as transient. + +data_dir: "/var/lib/vector" + +sources: + demo_logs: + type: "demo_logs" + format: "json" + interval: 1 + +sinks: + http_out: + type: "http" + inputs: ["demo_logs"] + uri: "https://example.com/ingest" + method: "post" + + # Skip the startup probe so the example can be adapted locally. + healthcheck: + enabled: false + + # Send newline-delimited JSON in the request body. + framing: + method: "newline_delimited" + encoding: + codec: "json" + + # Control how many retries are made and how quickly backoff grows. + request: + timeout_secs: 60 + retry_attempts: 8 + retry_initial_backoff_secs: 2 + retry_max_duration_secs: 30 + + # Retry only on the exact HTTP status codes that this destination + # treats as temporary failures. + retry_strategy: + type: "custom" + status_codes: [408, 425, 429, 503] diff --git a/lib/vector-config/src/http.rs b/lib/vector-config/src/http.rs index 94029610f0441..da51ea1cc156e 100644 --- a/lib/vector-config/src/http.rs +++ b/lib/vector-config/src/http.rs @@ -2,9 +2,11 @@ use std::cell::RefCell; use http::StatusCode; use serde_json::Value; +use vector_config_common::{attributes::CustomAttribute, constants}; use crate::{ Configurable, GenerateError, Metadata, ToValue, + num::NumberClass, schema::{SchemaGenerator, SchemaObject, generate_number_schema}, }; @@ -27,6 +29,10 @@ impl Configurable for StatusCode { let mut metadata = Metadata::default(); metadata.set_description("HTTP response status code"); metadata.set_default_value(StatusCode::OK); + metadata.add_custom_attribute(CustomAttribute::kv( + constants::DOCS_META_NUMERIC_TYPE, + NumberClass::Unsigned, + )); metadata } diff --git a/src/sinks/appsignal/config.rs b/src/sinks/appsignal/config.rs index 9e8c000f995a3..fea94af05b132 100644 --- a/src/sinks/appsignal/config.rs +++ b/src/sinks/appsignal/config.rs @@ -21,7 +21,7 @@ use crate::{ prelude::{SinkConfig, SinkContext}, util::{ BatchConfig, Compression, ServiceBuilderExt, SinkBatchSettings, TowerRequestConfig, - http::HttpStatusRetryLogic, + http::{HttpStatusRetryLogic, RetryStrategy}, }, }, }; @@ -67,6 +67,10 @@ pub(super) struct AppsignalConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + retry_strategy: RetryStrategy, } pub(super) fn default_endpoint() -> String { @@ -99,7 +103,10 @@ impl AppsignalConfig { let request_opts = self.request; let request_settings = request_opts.into_settings(); - let retry_logic = HttpStatusRetryLogic::new(|req: &AppsignalResponse| req.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |req: &AppsignalResponse| req.http_status, + self.retry_strategy.clone(), + ); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/axiom/config.rs b/src/sinks/axiom/config.rs index 61421973d5475..0930a997abc12 100644 --- a/src/sinks/axiom/config.rs +++ b/src/sinks/axiom/config.rs @@ -15,7 +15,8 @@ use crate::{ Healthcheck, VectorSink, http::config::{HttpMethod, HttpSinkConfig}, util::{ - BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, http::RequestConfig, + BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, + http::{RequestConfig, RetryStrategy}, }, }, tls::TlsConfig, @@ -124,6 +125,10 @@ pub struct AxiomConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl GenerateConfig for AxiomConfig { @@ -180,6 +185,7 @@ impl SinkConfig for AxiomConfig { ), payload_prefix: "".into(), // Always newline delimited JSON payload_suffix: "".into(), // Always newline delimited JSON + retry_strategy: self.retry_strategy.clone(), }; http_sink_config.build(cx).await diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 3b105d549d0c1..89da496045d9a 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -10,7 +10,10 @@ use crate::{ sinks::{ azure_common::config::AzureAuthentication, prelude::*, - util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, + util::{ + RealtimeSizeBasedDefaultBatchSettings, UriSerde, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, }; @@ -102,6 +105,10 @@ pub struct AzureLogsIngestionConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl Default for AzureLogsIngestionConfig { @@ -118,6 +125,7 @@ impl Default for AzureLogsIngestionConfig { request: Default::default(), tls: None, acknowledgements: Default::default(), + retry_strategy: Default::default(), } } } @@ -156,8 +164,10 @@ impl AzureLogsIngestionConfig { )?; let healthcheck = service.healthcheck(); - let retry_logic = - HttpStatusRetryLogic::new(|res: &AzureLogsIngestionResponse| res.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |res: &AzureLogsIngestionResponse| res.http_status, + self.retry_strategy.clone(), + ); let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/azure_monitor_logs/config.rs b/src/sinks/azure_monitor_logs/config.rs index dc60d179a2ff5..6d68aed7efd75 100644 --- a/src/sinks/azure_monitor_logs/config.rs +++ b/src/sinks/azure_monitor_logs/config.rs @@ -16,7 +16,10 @@ use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ prelude::*, - util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, + util::{ + RealtimeSizeBasedDefaultBatchSettings, UriSerde, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, }; @@ -113,6 +116,10 @@ pub struct AzureMonitorLogsConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl Default for AzureMonitorLogsConfig { @@ -129,6 +136,7 @@ impl Default for AzureMonitorLogsConfig { time_generated_key: None, tls: None, acknowledgements: Default::default(), + retry_strategy: Default::default(), } } } @@ -181,8 +189,10 @@ impl AzureMonitorLogsConfig { )?; let healthcheck = service.healthcheck(); - let retry_logic = - HttpStatusRetryLogic::new(|res: &AzureMonitorLogsResponse| res.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |res: &AzureMonitorLogsResponse| res.http_status, + self.retry_strategy.clone(), + ); let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/datadog/events/config.rs b/src/sinks/datadog/events/config.rs index 2bfdc66315c3e..f9b0f9361bba8 100644 --- a/src/sinks/datadog/events/config.rs +++ b/src/sinks/datadog/events/config.rs @@ -14,7 +14,10 @@ use crate::{ sinks::{ Healthcheck, VectorSink, datadog::{DatadogCommonConfig, LocalDatadogCommonConfig}, - util::{ServiceBuilderExt, TowerRequestConfig, http::HttpStatusRetryLogic}, + util::{ + ServiceBuilderExt, TowerRequestConfig, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, tls::MaybeTlsSettings, }; @@ -33,6 +36,10 @@ pub struct DatadogEventsConfig { #[configurable(derived)] #[serde(default)] pub request: TowerRequestConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl GenerateConfig for DatadogEventsConfig { @@ -64,7 +71,10 @@ impl DatadogEventsConfig { let request_opts = self.request; let request_settings = request_opts.into_settings(); - let retry_logic = HttpStatusRetryLogic::new(|req: &DatadogEventsResponse| req.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |req: &DatadogEventsResponse| req.http_status, + self.retry_strategy.clone(), + ); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/gcp/stackdriver/logs/config.rs b/src/sinks/gcp/stackdriver/logs/config.rs index 9b66eea4ac214..addb45163df76 100644 --- a/src/sinks/gcp/stackdriver/logs/config.rs +++ b/src/sinks/gcp/stackdriver/logs/config.rs @@ -21,7 +21,7 @@ use crate::{ prelude::*, util::{ BoxedRawValue, RealtimeSizeBasedDefaultBatchSettings, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, service::TowerRequestConfigDefaults, }, }, @@ -106,6 +106,10 @@ pub(super) struct StackdriverConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } pub(super) fn default_endpoint() -> String { @@ -269,7 +273,10 @@ impl SinkConfig for StackdriverConfig { let service = HttpService::new(client.clone(), stackdriver_logs_service_request_builder); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = StackdriverLogsSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/gcp/stackdriver/metrics/config.rs b/src/sinks/gcp/stackdriver/metrics/config.rs index 2398a18e9beb9..ea2cc9d6b5293 100644 --- a/src/sinks/gcp/stackdriver/metrics/config.rs +++ b/src/sinks/gcp/stackdriver/metrics/config.rs @@ -15,7 +15,8 @@ use crate::{ prelude::*, util::{ http::{ - HttpRequest, HttpService, HttpServiceRequestBuilder, http_response_retry_logic, + HttpRequest, HttpService, HttpServiceRequestBuilder, RetryStrategy, + http_response_retry_logic, }, service::TowerRequestConfigDefaults, }, @@ -77,6 +78,10 @@ pub struct StackdriverConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_metric_namespace_value() -> String { @@ -126,7 +131,10 @@ impl SinkConfig for StackdriverConfig { let service = HttpService::new(client, stackdriver_metrics_service_request_builder); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = StackdriverMetricsSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/honeycomb/config.rs b/src/sinks/honeycomb/config.rs index 05a56cd9df368..f179248e1697b 100644 --- a/src/sinks/honeycomb/config.rs +++ b/src/sinks/honeycomb/config.rs @@ -16,7 +16,7 @@ use crate::{ prelude::*, util::{ BatchConfig, BoxedRawValue, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, }, }, }; @@ -71,6 +71,10 @@ pub struct HoneycombConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_endpoint() -> String { @@ -124,7 +128,10 @@ impl SinkConfig for HoneycombConfig { let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = HoneycombSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index a3a105fa5c608..2b6abebe0d402 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -30,7 +30,10 @@ use crate::{ prelude::*, util::{ RealtimeSizeBasedDefaultBatchSettings, UriSerde, - http::{HttpService, OrderedHeaderName, RequestConfig, http_response_retry_logic}, + http::{ + HttpService, OrderedHeaderName, RequestConfig, RetryStrategy, + http_response_retry_logic, + }, }, }, }; @@ -100,6 +103,10 @@ pub struct HttpSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } /// HTTP method. @@ -330,7 +337,10 @@ impl SinkConfig for HttpSinkConfig { let request_limits = self.request.tower.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = HttpSink::new( @@ -405,6 +415,7 @@ mod tests { acknowledgements: AcknowledgementsConfig::default(), payload_prefix: String::new(), payload_suffix: String::new(), + retry_strategy: RetryStrategy::default(), }; let external_resource = ExternalResource::new( diff --git a/src/sinks/http/tests.rs b/src/sinks/http/tests.rs index 12ff994239f5d..2b4b6942c5c7b 100644 --- a/src/sinks/http/tests.rs +++ b/src/sinks/http/tests.rs @@ -67,6 +67,7 @@ fn default_cfg(encoding: EncodingConfigWithFraming) -> HttpSinkConfig { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), } } @@ -445,6 +446,105 @@ async fn retries_on_temporary_error() { .await; } +#[tokio::test] +async fn custom_retry_retries_only_configured_status_code() { + components::assert_sink_compliance(&HTTP_SINK_TAGS, async { + const NUM_LINES: usize = 1; + const NUM_FAILURES: usize = 2; + const CUSTOM_RETRY_CONFIG: &str = r#" + request.retry_attempts = 2 + request.retry_initial_backoff_secs = 1 + request.retry_max_duration_secs = 1 + retry_strategy.type = "custom" + retry_strategy.status_codes = [408, 425, 429, 503] + "#; + + let (in_addr, sink) = build_sink(CUSTOM_RETRY_CONFIG).await; + + let counter = Arc::new(atomic::AtomicUsize::new(0)); + let in_counter = Arc::clone(&counter); + let (rx, trigger, server) = build_test_server_generic(in_addr, move || { + let count = in_counter.fetch_add(1, atomic::Ordering::Relaxed); + if count < NUM_FAILURES { + Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::empty()) + .unwrap_or_else(|_| unreachable!()) + } else { + Response::new(Body::empty()) + } + }); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (input_lines, events) = random_lines_with_stream(100, NUM_LINES, Some(batch)); + let pump = sink.run(events); + + tokio::spawn(server); + + pump.await.unwrap(); + drop(trigger); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); + + let output_lines = get_received_gzip(rx, |parts| { + assert_eq!(Method::POST, parts.method); + assert_eq!("/frames", parts.uri.path()); + }) + .await; + + let tries = counter.load(atomic::Ordering::Relaxed); + assert_eq!(tries, NUM_FAILURES + 1); + assert_eq!(NUM_LINES, output_lines.len()); + assert_eq!(input_lines, output_lines); + }) + .await; +} + +#[tokio::test] +async fn custom_retry_does_not_retry_unconfigured_status_code() { + components::assert_sink_error(&COMPONENT_ERROR_TAGS, async { + const NUM_LINES: usize = 1; + const CUSTOM_RETRY_CONFIG: &str = r#" + request.retry_attempts = 2 + request.retry_initial_backoff_secs = 1 + request.retry_max_duration_secs = 1 + retry_strategy.type = "custom" + retry_strategy.status_codes = [408, 425, 429, 503] + "#; + + let (in_addr, sink) = build_sink(CUSTOM_RETRY_CONFIG).await; + + let counter = Arc::new(atomic::AtomicUsize::new(0)); + let in_counter = Arc::clone(&counter); + let (rx, trigger, server) = build_test_server_generic(in_addr, move || { + in_counter.fetch_add(1, atomic::Ordering::Relaxed); + Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::empty()) + .unwrap_or_else(|_| unreachable!()) + }); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (_input_lines, events) = random_lines_with_stream(100, NUM_LINES, Some(batch)); + let pump = sink.run(events); + + tokio::spawn(server); + + pump.await.unwrap(); + drop(trigger); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Rejected)); + assert_eq!(counter.load(atomic::Ordering::Relaxed), 1); + + let output_lines = get_received_gzip(rx, |_| { + unreachable!("There should be no successful requests") + }) + .await; + assert!(output_lines.is_empty()); + }) + .await; +} + #[tokio::test] async fn fails_on_permanent_error() { components::assert_sink_error(&COMPONENT_ERROR_TAGS, async { diff --git a/src/sinks/keep/config.rs b/src/sinks/keep/config.rs index 7b81281253672..d3810976aa3c1 100644 --- a/src/sinks/keep/config.rs +++ b/src/sinks/keep/config.rs @@ -16,7 +16,7 @@ use crate::{ prelude::*, util::{ BatchConfig, BoxedRawValue, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, }, }, }; @@ -59,6 +59,10 @@ pub struct KeepConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_endpoint() -> String { @@ -111,7 +115,10 @@ impl SinkConfig for KeepConfig { let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = KeepSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 7d73e25c4030d..dcc13624372d5 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -56,6 +56,7 @@ impl Default for Protocol { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), }) } } diff --git a/src/sinks/prometheus/remote_write/config.rs b/src/sinks/prometheus/remote_write/config.rs index c1b420d6b216d..c9f214953003c 100644 --- a/src/sinks/prometheus/remote_write/config.rs +++ b/src/sinks/prometheus/remote_write/config.rs @@ -17,7 +17,7 @@ use crate::{ prometheus::PrometheusRemoteWriteAuth, util::{ auth::Auth, - http::{OrderedHeaderName, http_response_retry_logic}, + http::{OrderedHeaderName, RetryStrategy, http_response_retry_logic}, service::TowerRequestConfig, }, }, @@ -129,6 +129,10 @@ pub struct RemoteWriteConfig { #[serde(default = "default_compression")] #[derivative(Default(value = "default_compression()"))] pub compression: Compression, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } const fn default_compression() -> Compression { @@ -251,7 +255,10 @@ impl SinkConfig for RemoteWriteConfig { headers: validated_headers, }; let service = ServiceBuilder::new() - .settings(request_settings, http_response_retry_logic()) + .settings( + request_settings, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = RemoteWriteSink { diff --git a/src/sinks/util/http.rs b/src/sinks/util/http.rs index e19025e3c6ee3..89225f4077741 100644 --- a/src/sinks/util/http.rs +++ b/src/sinks/util/http.rs @@ -7,6 +7,7 @@ use futures::{Sink, future::BoxFuture}; use headers::HeaderName; use http::{HeaderValue, Request, Response, StatusCode, header}; use http_body::Body as _; +use tracing::debug; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OrderedHeaderName(HeaderName); @@ -550,14 +551,137 @@ impl sink::Response for http::Response { } } +/// Serializes and deserializes a [`Vec`] +mod status_code_vec { + use http::StatusCode; + use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; + + /// Deserializes a [`Vec`] + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Vec::::deserialize(deserializer)? + .into_iter() + .map(StatusCode::from_u16) + .collect::, _>>() + .map_err(Error::custom) + } + + /// Serializes a [`Vec`] + pub fn serialize(status_codes: &[StatusCode], serializer: S) -> Result + where + S: Serializer, + { + status_codes + .iter() + .map(StatusCode::as_u16) + .collect::>() + .serialize(serializer) + } +} + +/// Configurable retry strategy for `http` based sinks. +/// +/// For more information about error responses, see [Client Error Responses][error_responses]. +/// +/// [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses +#[configurable_component] +#[derive(Debug, Clone, Default, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "The retry strategy enum."))] +pub enum RetryStrategy { + /// Don't retry any errors, including request timeouts. + None, + + /// Default strategy. See [`RetryStrategy::retry_action`] for more details. + #[default] + Default, + + /// Retry on *all* HTTP status codes except for success codes (2xx) + All, + + /// Custom retry strategy + Custom { + /// Retry on these specific HTTP status codes + #[serde(with = "status_code_vec")] + status_codes: Vec, + }, +} + +impl RetryStrategy { + /// Returns the name of the retry strategy. + #[must_use] + const fn name(&self) -> &str { + match self { + Self::None => "Never retry strategy", + Self::Default => "Default retry strategy", + Self::All => "Retry all strategy", + Self::Custom { .. } => "Custom retry strategy", + } + } + + /// Determines if the given status code should be retried. + /// + /// For the `Default` strategy, the following status codes will be retried: + /// - 429 (Too Many Requests) + /// - 408 (Request Timeout) + /// - 5xx (Server Error) + /// + /// For the `Custom` strategy, the status codes specified in the `status_codes` field will be retried. + /// + /// For the `All` strategy, all non-success status codes will be retried. + #[must_use] + pub fn retry_action(&self, status: http::StatusCode) -> RetryAction { + if status.is_success() { + return RetryAction::Successful; + } + + let reason = format!( + "{}: {}", + self.name(), + status.canonical_reason().unwrap_or_else(|| status.as_str()) + ) + .into(); + + match self { + Self::None => RetryAction::DontRetry(reason), + Self::Default => match status { + StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT => { + RetryAction::Retry(reason) + } + StatusCode::NOT_IMPLEMENTED => RetryAction::DontRetry(reason), + _ => { + if status.is_server_error() { + RetryAction::Retry(reason) + } else { + RetryAction::DontRetry(reason) + } + } + }, + Self::All => RetryAction::Retry(reason), + Self::Custom { status_codes } => { + if status_codes.contains(&status) { + RetryAction::Retry(reason) + } else { + RetryAction::DontRetry(reason) + } + } + } + } +} + #[derive(Debug, Clone)] pub struct HttpRetryLogic { request: PhantomData, + retry_strategy: RetryStrategy, } + impl Default for HttpRetryLogic { fn default() -> Self { Self { request: PhantomData, + retry_strategy: RetryStrategy::Default, } } } @@ -567,25 +691,28 @@ impl RetryLogic for HttpRetryLogic { type Request = Req; type Response = hyper::Response; - fn is_retriable_error(&self, _error: &Self::Error) -> bool { - true + fn is_retriable_error(&self, error: &Self::Error) -> bool { + if self.retry_strategy == RetryStrategy::None { + false + } else { + error.is_retriable() + } + } + + fn is_retriable_timeout(&self) -> bool { + self.retry_strategy != RetryStrategy::None } fn should_retry_response(&self, response: &Self::Response) -> RetryAction { let status = response.status(); - - match status { - StatusCode::TOO_MANY_REQUESTS => RetryAction::Retry("too many requests".into()), - StatusCode::REQUEST_TIMEOUT => RetryAction::Retry("request timeout".into()), - StatusCode::NOT_IMPLEMENTED => { - RetryAction::DontRetry("endpoint not implemented".into()) - } - _ if status.is_server_error() => RetryAction::Retry( - format!("{}: {}", status, String::from_utf8_lossy(response.body())).into(), - ), - _ if status.is_success() => RetryAction::Successful, - _ => RetryAction::DontRetry(format!("response status: {status}").into()), + if !status.is_success() { + debug!( + message = "HTTP response.", + %status, + body = %String::from_utf8_lossy(response.body()), + ); } + self.retry_strategy.retry_action(status) } } @@ -596,6 +723,7 @@ pub struct HttpStatusRetryLogic { func: F, request: PhantomData, response: PhantomData, + retry_strategy: RetryStrategy, } impl HttpStatusRetryLogic @@ -604,11 +732,12 @@ where Req: Send + Sync + 'static, Res: Send + Sync + 'static, { - pub const fn new(func: F) -> HttpStatusRetryLogic { + pub const fn new(func: F, retry_strategy: RetryStrategy) -> HttpStatusRetryLogic { HttpStatusRetryLogic { func, request: PhantomData, response: PhantomData, + retry_strategy, } } } @@ -623,25 +752,21 @@ where type Request = Req; type Response = Res; - fn is_retriable_error(&self, _error: &Self::Error) -> bool { - true + fn is_retriable_error(&self, error: &Self::Error) -> bool { + if self.retry_strategy == RetryStrategy::None { + false + } else { + error.is_retriable() + } + } + + fn is_retriable_timeout(&self) -> bool { + self.retry_strategy != RetryStrategy::None } fn should_retry_response(&self, response: &Res) -> RetryAction { let status = (self.func)(response); - - match status { - StatusCode::TOO_MANY_REQUESTS => RetryAction::Retry("too many requests".into()), - StatusCode::REQUEST_TIMEOUT => RetryAction::Retry("request timeout".into()), - StatusCode::NOT_IMPLEMENTED => { - RetryAction::DontRetry("endpoint not implemented".into()) - } - _ if status.is_server_error() => { - RetryAction::Retry(format!("Http Status: {status}").into()) - } - _ if status.is_success() => RetryAction::Successful, - _ => RetryAction::DontRetry(format!("Http status: {status}").into()), - } + self.retry_strategy.retry_action(status) } } @@ -654,6 +779,7 @@ where func: self.func.clone(), request: PhantomData, response: PhantomData, + retry_strategy: self.retry_strategy.clone(), } } } @@ -820,12 +946,17 @@ impl DriverResponse for HttpResponse { } /// Creates a `RetryLogic` for use with `HttpResponse`. -pub fn http_response_retry_logic() -> HttpStatusRetryLogic< +pub fn http_response_retry_logic( + retry_strategy: RetryStrategy, +) -> HttpStatusRetryLogic< impl Fn(&HttpResponse) -> StatusCode + Clone + Send + Sync + 'static, Request, HttpResponse, > { - HttpStatusRetryLogic::new(|req: &HttpResponse| req.http_response.status()) + HttpStatusRetryLogic::new( + |req: &HttpResponse| req.http_response.status(), + retry_strategy, + ) } /// Uses the estimated json encoded size to determine batch sizing. @@ -969,6 +1100,93 @@ mod test { ); } + #[test] + fn retry_strategy_none_preserves_success_and_rejects_failures() { + let strategy = RetryStrategy::None; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_not_retryable() + ); + } + + #[test] + fn retry_strategy_none_disables_timeout_retries() { + let logic = HttpRetryLogic::<()> { + request: PhantomData, + retry_strategy: RetryStrategy::None, + }; + let status_logic = + HttpStatusRetryLogic::<_, (), ()>::new(|_: &()| StatusCode::OK, RetryStrategy::None); + + assert!(!logic.is_retriable_timeout()); + assert!(!status_logic.is_retriable_timeout()); + } + + #[test] + fn retry_strategy_all_preserves_success_and_retries_failures() { + let strategy = RetryStrategy::All; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::BAD_REQUEST) + .is_retryable() + ); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_retryable() + ); + } + + #[test] + fn retry_strategy_custom_only_retries_configured_statuses() { + let strategy = RetryStrategy::Custom { + status_codes: vec![StatusCode::BAD_REQUEST], + }; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::BAD_REQUEST) + .is_retryable() + ); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_not_retryable() + ); + } + + #[test] + fn retry_strategy_custom_serde_roundtrips_status_codes() { + let json = r#"{"type":"custom","status_codes":[400,503]}"#; + let strategy: RetryStrategy = serde_json::from_str(json).unwrap(); + assert_eq!( + strategy, + RetryStrategy::Custom { + status_codes: vec![StatusCode::BAD_REQUEST, StatusCode::SERVICE_UNAVAILABLE], + } + ); + let encoded = serde_json::to_string(&strategy).unwrap(); + let roundtrip: RetryStrategy = serde_json::from_str(&encoded).unwrap(); + assert_eq!(roundtrip, strategy); + } + + #[test] + fn retry_strategy_custom_serde_rejects_invalid_status_codes() { + // `http::StatusCode::from_u16` only accepts 100–999; 1000 is out of range. + let json = r#"{"type":"custom","status_codes":[1000]}"#; + let result = serde_json::from_str::(json); + assert!( + result.is_err(), + "expected invalid status code to fail deserialization" + ); + } + #[tokio::test] async fn util_http_it_makes_http_requests() { let (_guard, addr) = next_addr(); diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index be05d559bb18c..2a38a2b4ee900 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -34,6 +34,12 @@ pub trait RetryLogic: Clone + Send + Sync + 'static { /// implementors to specify what kinds of errors can be retried. fn is_retriable_error(&self, error: &Self::Error) -> bool; + /// When the Service call times out, this function allows implementors to + /// specify if the timeout should be retried. + fn is_retriable_timeout(&self) -> bool { + true + } + /// When the Service call returns an `Ok` response, this function allows /// implementors to specify additional logic to determine if the success response /// is actually an error. This is particularly useful when the downstream service @@ -197,10 +203,18 @@ where None } } else if error.downcast_ref::().is_some() { - warn!( - "Request timed out. If this happens often while the events are actually reaching their destination, try decreasing `batch.max_bytes` and/or using `compression` if applicable. Alternatively `request.timeout_secs` can be increased." - ); - Some(self.build_retry()) + if self.logic.is_retriable_timeout() { + warn!( + "Request timed out. If this happens often while the events are actually reaching their destination, try decreasing `batch.max_bytes` and/or using `compression` if applicable. Alternatively `request.timeout_secs` can be increased." + ); + Some(self.build_retry()) + } else { + error!( + message = + "Request timed out and is not retriable; dropping the request." + ); + None + } } else { error!( message = "Unexpected error type; dropping the request.", @@ -338,6 +352,27 @@ mod tests { assert_eq!(fut.await.unwrap(), "world"); } + #[tokio::test] + async fn timeout_error_no_retry() { + trace_init(); + + let policy = FibonacciRetryPolicy::new( + 5, + Duration::from_secs(1), + Duration::from_secs(10), + NoTimeoutRetryLogic, + JitterMode::None, + ); + + let (mut svc, mut handle) = mock::spawn_layer(RetryLayer::new(policy)); + + assert_ready_ok!(svc.poll_ready()); + + let mut fut = task::spawn(svc.call("hello")); + assert_request_eq!(handle, "hello").send_error(Elapsed::new()); + assert_ready_err!(fut.poll()); + } + #[test] fn backoff_grows_to_max() { let mut policy = FibonacciRetryPolicy::new( @@ -425,6 +460,23 @@ mod tests { } } + #[derive(Debug, Clone)] + struct NoTimeoutRetryLogic; + + impl RetryLogic for NoTimeoutRetryLogic { + type Error = Error; + type Request = &'static str; + type Response = &'static str; + + fn is_retriable_error(&self, error: &Self::Error) -> bool { + error.0 + } + + fn is_retriable_timeout(&self) -> bool { + false + } + } + #[derive(Debug)] struct Error(bool); diff --git a/website/cue/reference/components/sinks/generated/appsignal.cue b/website/cue/reference/components/sinks/generated/appsignal.cue index eb3710974335f..47943be3c1f93 100644 --- a/website/cue/reference/components/sinks/generated/appsignal.cue +++ b/website/cue/reference/components/sinks/generated/appsignal.cue @@ -323,6 +323,37 @@ generated: components: sinks: appsignal: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "Configures the TLS options for incoming/outgoing connections." required: false diff --git a/website/cue/reference/components/sinks/generated/axiom.cue b/website/cue/reference/components/sinks/generated/axiom.cue index d4dc62e6294fc..a1f42404ac0c1 100644 --- a/website/cue/reference/components/sinks/generated/axiom.cue +++ b/website/cue/reference/components/sinks/generated/axiom.cue @@ -305,6 +305,37 @@ generated: components: sinks: axiom: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: """ The TLS settings for the connection. diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 2aaf9b628e3c6..68944f5a6e7d5 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -408,6 +408,37 @@ generated: components: sinks: azure_logs_ingestion: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } stream_name: { description: """ The [Stream name][stream_name] for the Data collection rule. diff --git a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue index 98b14f8b4faad..c7039b70097e1 100644 --- a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue @@ -314,6 +314,37 @@ generated: components: sinks: azure_monitor_logs: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } shared_key: { description: """ The [primary or the secondary key][shared_key] for the Log Analytics workspace. diff --git a/website/cue/reference/components/sinks/generated/datadog_events.cue b/website/cue/reference/components/sinks/generated/datadog_events.cue index 7316613acef4e..23b9fb43d6413 100644 --- a/website/cue/reference/components/sinks/generated/datadog_events.cue +++ b/website/cue/reference/components/sinks/generated/datadog_events.cue @@ -242,6 +242,37 @@ generated: components: sinks: datadog_events: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } site: { description: """ The Datadog [site][dd_site] to send observability data to. diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue index 2909192f14d3f..9d5e76262bfd2 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue @@ -420,6 +420,37 @@ generated: components: sinks: gcp_stackdriver_logs: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } severity_key: { description: """ The field of the log event from which to take the outgoing log’s `severity` field. diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue index 23ef271177f3a..52c344e278250 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue @@ -334,6 +334,37 @@ generated: components: sinks: gcp_stackdriver_metrics: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/generated/honeycomb.cue b/website/cue/reference/components/sinks/generated/honeycomb.cue index 66ba9ed795a27..c65064a135a00 100644 --- a/website/cue/reference/components/sinks/generated/honeycomb.cue +++ b/website/cue/reference/components/sinks/generated/honeycomb.cue @@ -321,4 +321,35 @@ generated: components: sinks: honeycomb: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } } diff --git a/website/cue/reference/components/sinks/generated/http.cue b/website/cue/reference/components/sinks/generated/http.cue index 0fcc53fab8092..8523a27dd0d9b 100644 --- a/website/cue/reference/components/sinks/generated/http.cue +++ b/website/cue/reference/components/sinks/generated/http.cue @@ -1024,6 +1024,37 @@ generated: components: sinks: http: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/generated/keep.cue b/website/cue/reference/components/sinks/generated/keep.cue index 29b1d7536611b..d2ed08114a2a9 100644 --- a/website/cue/reference/components/sinks/generated/keep.cue +++ b/website/cue/reference/components/sinks/generated/keep.cue @@ -286,4 +286,35 @@ generated: components: sinks: keep: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } } diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 4505d8a9bf512..4e8e9d9ff7210 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1025,6 +1025,37 @@ generated: components: sinks: opentelemetry: configuration: protocol: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue index c13fa424999ec..5d74d52359bdd 100644 --- a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue @@ -539,6 +539,37 @@ generated: components: sinks: prometheus_remote_write: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tenant_id: { description: """ The tenant ID to send. From 9c617a7b766dc95ea919384ff16da2654595f6a4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 1 May 2026 10:04:29 -0400 Subject: [PATCH 135/364] fix(ci): grant issues:write to remove_wip_label workflow (#25339) The github.rest.issues.removeLabel call (DELETE /repos/{owner}/{repo}/issues/{issue_number}/labels/{name}) requires the issues:write permission per the GitHub REST API. The workflow only declared pull-requests:write, so the call returned 403 the first time all preconditions actually held simultaneously (MEMBER reviewer + label present + approval). Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/remove_wip_label.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_wip_label.yml index 7c30ee892a949..b4750b0f2b0d6 100644 --- a/.github/workflows/remove_wip_label.yml +++ b/.github/workflows/remove_wip_label.yml @@ -1,6 +1,7 @@ name: Remove Work In Progress Label permissions: + issues: write pull-requests: write on: From 308d2469b2e70153d3eafd20a647bf33a9d69d73 Mon Sep 17 00:00:00 2001 From: tot19 <31141271+tot19@users.noreply.github.com> Date: Fri, 1 May 2026 23:43:39 +0900 Subject: [PATCH 136/364] fix(sources): prevent windows_event_log permanent freeze from signal-event lost wakeup (#25195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(windows_event_log): fix pre-drain ResetEvent race and add lost-wakeup regression tests The Windows Event Log service signals the pull-mode wait handle via SetEvent each time a new matching event is recorded. Because the handle is manual-reset, any SetEvent that fires between the last EvtNext call and the post-drain ResetEvent is silently lost — the subscription then hangs until the next OS event arrives (vectordotdev/vector#25194). Fix: reset the handle *before* entering the drain loop. Signals raised during the drain are preserved because SetEvent on an already-signaled handle is a no-op. Re-arm (SetEvent) on early exits so the next pull_events revisits the channel without waiting for a fresh OS notification: - budget exhaustion - bookmark failure mid-batch - transient EvtNext error Regression tests: - test_pull_events_preserves_setevent_during_drain: installs DRAIN_STEP_HOOK to fire SetEvent mid-drain and asserts wait_for_events_blocking returns EventsAvailable, not Timeout. - test_speculative_pull_recovers_without_signal: manually clears the channel signal via ResetEvent, confirms wait times out, then asserts pull_events still returns events — proving the speculative timeout pull in mod.rs self-heals independently of signal state. Also: comment re-subscription break paths (ERROR_EVT_QUERY_RESULT_STALE and INVALID_POSITION) noting the speculative pull as a safety net if the re-subbed channel does not immediately re-signal; add serialization note to DRAIN_STEP_HOOK. * fix(windows_event_log): add speculative timeout pull, deduplicate processing, fix error handling Four related improvements to mod.rs: 1. Speculative pull on WaitResult::Timeout: call pull_events on every timeout cycle as a belt-and-suspenders self-heal. EvtNext returns ERROR_NO_MORE_ITEMS immediately on an empty channel (near-zero cost). If events are recovered a warning is emitted. Guarantees recovery within one timeout period regardless of the root cause of the lost wakeup. 2. Extract with_subscription_blocking helper: wraps the spawn_blocking ownership-transfer pattern (move subscription in, run blocking fn, return subscription + result). All three blocking calls (wait, normal pull, speculative pull) now use this helper instead of inlining spawn_blocking. 3. Extract process_event_batch helper: the parse/emit/send_batch/finalize sequence was duplicated verbatim between the EventsAvailable arm and the speculative-timeout arm. Extracted into a shared free async function. Rate limiting is applied consistently in both paths via the helper. 4. Fix error-handling asymmetry: the speculative pull Err branch previously only logged warn! and continued, so a non-recoverable error (access denied, channel not found) would spam warnings indefinitely. Now mirrors the EventsAvailable path: emit WindowsEventLogQueryError, break on non-recoverable errors, apply exponential backoff on recoverable ones. * test(windows_event_log source): harden lost-wakeup regression tests against flakiness Address two independent flakiness sources in the #25194 regression tests so the suite is stable on real Windows CI runners. test_pull_events_preserves_setevent_during_drain: - Replaced a 1000ms blocking wait with an immediate 0ms poll after pull_events returns, so the check measures only the reset/preserve behavior of pull_events and is not contaminated by unrelated Windows system events signaling the handle during a nonzero wait window. - Keyed the DRAIN_STEP_HOOK fire-once to the subscription's own signal handle so concurrent pull_events calls from other tests can't flip the hook first and SetEvent the wrong handle. test_speculative_pull_recovers_without_signal: - Same 500ms→0ms poll change, opposite direction: real events arriving during the wait would have re-signaled the manually-cleared handle and flipped the expected Timeout into a real signal result. - Seed a deterministic record via 'eventcreate' before subscription creation so the non-empty-events assertion is independent of whatever backlog the runner happens to have. Freshly provisioned images can have an empty Application log, which would otherwise cause pull_events(100) to legitimately return empty and false-fail the test. * fix(windows_event_log): implement Sync for subscription types to satisfy Send bound on source future EventLogSubscription and ChannelSubscription had unsafe impl Send but no Sync impl. Since &T: Send requires T: Sync, process_event_batch holding &EventLogSubscription across an .await made the entire source future !Send, causing a compile error (ICE + future-not-Send) in release builds. All mutation on these types requires &mut self; &self methods are read-only or delegate to already-Sync types (RateLimiter). The underlying Windows kernel handles are safe for concurrent access. Co-Authored-By: Claude Sonnet 4.6 * chore(changelog): simplify windows_event_log fix fragment to user-focused one-liner Co-Authored-By: Claude Sonnet 4.6 * fix(windows_event_log): prioritize shutdown signal * fix(windows_event_log): lighten speculative timeout pulls --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Pavlos Rontidis --- ...25194_windows_event_log_lost_wakeup.fix.md | 3 + src/sources/windows_event_log/mod.rs | 281 ++++++++----- src/sources/windows_event_log/subscription.rs | 374 ++++++++++++++++-- 3 files changed, 546 insertions(+), 112 deletions(-) create mode 100644 changelog.d/25194_windows_event_log_lost_wakeup.fix.md diff --git a/changelog.d/25194_windows_event_log_lost_wakeup.fix.md b/changelog.d/25194_windows_event_log_lost_wakeup.fix.md new file mode 100644 index 0000000000000..cb6f67987d67c --- /dev/null +++ b/changelog.d/25194_windows_event_log_lost_wakeup.fix.md @@ -0,0 +1,3 @@ +The `windows_event_log` source no longer freezes after periods of inactivity. + +authors: tot19 diff --git a/src/sources/windows_event_log/mod.rs b/src/sources/windows_event_log/mod.rs index 17d900cc07622..28dfc1db4cb79 100644 --- a/src/sources/windows_event_log/mod.rs +++ b/src/sources/windows_event_log/mod.rs @@ -29,7 +29,7 @@ cfg_if::cfg_if! { use vector_lib::EstimatedJsonEncodedSizeOf; use vector_lib::finalizer::OrderedFinalizer; use vector_lib::internal_event::{ - ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol, + ByteSize, BytesReceived, CountByteSize, InternalEventHandle, Protocol, }; use windows::Win32::Foundation::{DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE}; use windows::Win32::System::Threading::GetCurrentProcess; @@ -48,6 +48,7 @@ cfg_if::cfg_if! { error::WindowsEventLogError, parser::EventLogParser, subscription::{EventLogSubscription, WaitResult}, + xml_parser::WindowsEvent, }; } } @@ -157,6 +158,107 @@ impl Finalizer { } } +/// Parse, emit metrics for, send, and finalize a non-empty batch of pulled Windows events. +/// +/// Both the `EventsAvailable` path and the speculative-timeout path share this +/// logic. Returns `true` if the downstream pipeline closed and the caller +/// should break out of the main event loop. +async fn process_event_batch( + events: Vec, + parser: &EventLogParser, + acknowledgements: bool, + subscription: &EventLogSubscription, + out: &mut SourceSender, + finalizer: &Finalizer, + events_received: &impl InternalEventHandle, + bytes_received: &impl InternalEventHandle, +) -> bool { + // Rate limiting between batches (async-compatible). + if let Some(limiter) = subscription.rate_limiter() { + limiter.until_ready().await; + } + + let (batch, receiver) = BatchNotifier::maybe_new_with_receiver(acknowledgements); + let mut log_events = Vec::new(); + let mut total_byte_size = 0usize; + let mut channels_in_batch = std::collections::HashSet::new(); + + for event in events { + let channel = event.channel.clone(); + channels_in_batch.insert(channel.clone()); + let event_id = event.event_id; + match parser.parse_event(event) { + Ok(mut log_event) => { + let byte_size = log_event.estimated_json_encoded_size_of(); + total_byte_size += byte_size.get(); + if let Some(ref batch) = batch { + log_event = log_event.with_batch_notifier(batch); + } + log_events.push(log_event); + } + Err(e) => { + emit!(WindowsEventLogParseError { + error: e.to_string(), + channel, + event_id: Some(event_id), + }); + } + } + } + + if !log_events.is_empty() { + let count = log_events.len(); + events_received.emit(CountByteSize(count, total_byte_size.into())); + bytes_received.emit(ByteSize(total_byte_size)); + + // BACK PRESSURE: block until the pipeline accepts the batch. + // We don't call EvtNext again until this completes. + if let Err(_error) = out.send_batch(log_events).await { + emit!(StreamClosedError { count }); + return true; // signal: break the main loop + } + + // Register checkpoint entry with the finalizer. + let bookmarks: Vec<(String, String)> = channels_in_batch + .into_iter() + .filter_map(|channel| { + subscription + .get_bookmark_xml(&channel) + .map(|xml| (channel, xml)) + }) + .collect(); + + if !bookmarks.is_empty() { + let entry = FinalizerEntry { bookmarks }; + finalizer.finalize(entry, receiver).await; + } + } + + false // pipeline still open +} + +/// Transfer ownership of `subscription` into a `spawn_blocking` task, run `f` +/// on it, then return both the subscription and the result. +/// +/// All blocking Windows APIs (`WaitForMultipleObjects`, `EvtNext`, `EvtRender`) +/// must run in `spawn_blocking` to avoid stalling the async runtime. The +/// ownership-transfer pattern ensures only one thread holds the subscription +/// at a time, preventing data races without requiring locks. +async fn with_subscription_blocking( + subscription: EventLogSubscription, + f: F, +) -> Result<(EventLogSubscription, R), WindowsEventLogError> +where + F: FnOnce(EventLogSubscription) -> (EventLogSubscription, R) + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(move || f(subscription)) + .await + .map_err(|e| WindowsEventLogError::ConfigError { + message: format!("Blocking subscription task panicked: {e}"), + }) +} + /// Windows Event Log source implementation pub struct WindowsEventLogSource { config: WindowsEventLogConfig, @@ -281,42 +383,25 @@ impl WindowsEventLogSource { // Ownership transfer ensures no data races between the blocking thread // and async code. The shutdown watcher uses a raw HANDLE value (just an // integer) to signal shutdown without needing access to the subscription. - let (returned_sub, wait_result) = tokio::task::spawn_blocking({ - let sub = subscription; - move || { + let (returned_sub, wait_result) = + with_subscription_blocking(subscription, move |sub| { let result = sub.wait_for_events_blocking(timeout_ms); (sub, result) - } - }) - .await - .map_err(|e| WindowsEventLogError::ConfigError { - message: format!("Wait task panicked: {e}"), - })?; - + }) + .await?; subscription = returned_sub; match wait_result { WaitResult::EventsAvailable => { // Pull events via spawn_blocking (EvtNext/EvtRender are blocking APIs) - let (returned_sub, events_result) = tokio::task::spawn_blocking({ - let mut sub = subscription; - move || { + let (returned_sub, events_result) = + with_subscription_blocking(subscription, move |mut sub| { let result = sub.pull_events(batch_size); (sub, result) - } - }) - .await - .map_err(|e| WindowsEventLogError::ConfigError { - message: format!("Pull task panicked: {e}"), - })?; - + }) + .await?; subscription = returned_sub; - // Rate limiting between batches (async-compatible) - if let Some(limiter) = subscription.rate_limiter() { - limiter.until_ready().await; - } - match events_result { Ok(events) if events.is_empty() => { error_backoff = std::time::Duration::from_millis(100); @@ -328,65 +413,19 @@ impl WindowsEventLogSource { message = "Pulled Windows Event Log events.", event_count = events.len() ); - - let (batch, receiver) = - BatchNotifier::maybe_new_with_receiver(acknowledgements); - - let mut log_events = Vec::new(); - let mut total_byte_size = 0; - let mut channels_in_batch = std::collections::HashSet::new(); - - for event in events { - let channel = event.channel.clone(); - channels_in_batch.insert(channel.clone()); - let event_id = event.event_id; - match parser.parse_event(event) { - Ok(mut log_event) => { - let byte_size = log_event.estimated_json_encoded_size_of(); - total_byte_size += byte_size.get(); - - if let Some(ref batch) = batch { - log_event = log_event.with_batch_notifier(batch); - } - - log_events.push(log_event); - } - Err(e) => { - emit!(WindowsEventLogParseError { - error: e.to_string(), - channel, - event_id: Some(event_id), - }); - } - } - } - - if !log_events.is_empty() { - let count = log_events.len(); - events_received.emit(CountByteSize(count, total_byte_size.into())); - bytes_received.emit(ByteSize(total_byte_size)); - - // BACK PRESSURE: block here until the pipeline accepts - // the batch. We don't call EvtNext again until this completes. - if let Err(_error) = out.send_batch(log_events).await { - emit!(StreamClosedError { count }); - break; - } - - // Register checkpoint entry with finalizer - let bookmarks: Vec<(String, String)> = channels_in_batch - .into_iter() - .filter_map(|channel| { - subscription - .get_bookmark_xml(&channel) - .map(|xml| (channel, xml)) - }) - .collect(); - - if !bookmarks.is_empty() { - let entry = FinalizerEntry { bookmarks }; - finalizer.finalize(entry, receiver).await; - } + if process_event_batch( + events, + &parser, + acknowledgements, + &subscription, + &mut out, + &finalizer, + &events_received, + &bytes_received, + ) + .await + { + break; } } Err(e) => { @@ -415,10 +454,6 @@ impl WindowsEventLogSource { } WaitResult::Timeout => { - // A full wait cycle without errors means the system is healthy; - // reset backoff so the next transient error starts fresh. - error_backoff = std::time::Duration::from_millis(100); - // Periodic checkpoint flush (sync mode only) if !acknowledgements && last_checkpoint.elapsed() >= checkpoint_interval { if let Err(e) = subscription.flush_bookmarks().await { @@ -448,6 +483,74 @@ impl WindowsEventLogSource { ); } } + + // Speculative pull: self-heal against any lost-wakeup scenario, + // regardless of root cause. If the OS signal was lost through any + // mechanism (not just the pre-drain race fixed in #25194), this + // ensures the source recovers within one timeout period. + // Use the speculative pull variant so idle timeout cycles don't + // refresh per-channel record-count gauges via EvtOpenLog / + // EvtGetLogInfo on every configured channel. + let (returned_sub, speculative_result) = + with_subscription_blocking(subscription, move |mut sub| { + let result = sub.pull_events_speculative(batch_size); + (sub, result) + }) + .await?; + subscription = returned_sub; + + match speculative_result { + Ok(events) if events.is_empty() => { + // Healthy cycle: reset backoff so the next transient + // error starts fresh. + error_backoff = std::time::Duration::from_millis(100); + } + Ok(events) => { + // Healthy cycle: reset backoff so the next transient + // error starts fresh. + error_backoff = std::time::Duration::from_millis(100); + warn!( + message = "Speculative timeout pull recovered events; possible lost wakeup detected.", + event_count = events.len(), + ); + if process_event_batch( + events, + &parser, + acknowledgements, + &subscription, + &mut out, + &finalizer, + &events_received, + &bytes_received, + ) + .await + { + break; + } + } + Err(e) => { + emit!(WindowsEventLogQueryError { + channel: "all".to_string(), + query: None, + error: e.to_string(), + }); + if !e.is_recoverable() { + error!( + message = "Non-recoverable speculative pull error, shutting down.", + error = %e + ); + break; + } + // Exponential backoff mirrors the EventsAvailable error path. + warn!( + message = "Recoverable speculative pull error, backing off.", + backoff_ms = error_backoff.as_millis() as u64, + error = %e + ); + tokio::time::sleep(error_backoff).await; + error_backoff = (error_backoff * 2).min(MAX_ERROR_BACKOFF); + } + } } WaitResult::Shutdown => { diff --git a/src/sources/windows_event_log/subscription.rs b/src/sources/windows_event_log/subscription.rs index dc92713f691d9..4571561a2d349 100644 --- a/src/sources/windows_event_log/subscription.rs +++ b/src/sources/windows_event_log/subscription.rs @@ -18,9 +18,9 @@ use windows::Win32::System::EventLog::{ EvtSubscribeStartAfterBookmark, EvtSubscribeStartAtOldestRecord, EvtSubscribeStrict, EvtSubscribeToFutureEvents, }; -#[cfg(test)] -use windows::Win32::System::Threading::SetEvent; -use windows::Win32::System::Threading::{CreateEventW, ResetEvent, WaitForMultipleObjects}; +use windows::Win32::System::Threading::{ + CreateEventW, ResetEvent, SetEvent, WaitForMultipleObjects, +}; use windows::core::HSTRING; use super::{ @@ -30,6 +30,19 @@ use super::{ use crate::internal_events::WindowsEventLogBookmarkError; +/// Test-only hook called inside the `pull_events` drain loop after each +/// `EvtNext` invocation. Used by the lost-wakeup regression test +/// (see `test_pull_events_preserves_setevent_during_drain`) to race a +/// `SetEvent` against the drain without relying on thread-timing. +/// No-op and zero-cost in non-test builds. +/// +/// Only one test should install a hook at a time; tests that install a hook +/// must use `#[serial_test::serial]` or equivalent serialization to prevent +/// concurrent tests from triggering each other's hook. +#[cfg(test)] +static DRAIN_STEP_HOOK: std::sync::Mutex>> = + std::sync::Mutex::new(None); + /// Maximum number of entries in the EvtFormatMessage result cache. pub const FORMAT_CACHE_CAPACITY: usize = 10_000; /// Maximum number of cached publisher metadata handles. @@ -80,6 +93,7 @@ struct ChannelSubscription { // SAFETY: Same rationale as EventLogSubscription - Windows kernel handles are thread-safe. unsafe impl Send for ChannelSubscription {} +unsafe impl Sync for ChannelSubscription {} /// Result of waiting for events across all channels. pub enum WaitResult { @@ -130,8 +144,10 @@ pub struct EventLogSubscription { // SAFETY: Windows HANDLE and EVT_HANDLE are kernel objects safe to use across // threads. In windows 0.58, HANDLE wraps *mut c_void which is !Send/!Sync, -// but the underlying kernel handles are thread-safe. +// but the underlying kernel handles are thread-safe. All mutation requires +// &mut self; &self methods are read-only or delegate to Sync types (RateLimiter). unsafe impl Send for EventLogSubscription {} +unsafe impl Sync for EventLogSubscription {} impl EventLogSubscription { /// Create a new pull-model subscription for all configured channels. @@ -415,21 +431,20 @@ impl EventLogSubscription { /// Wait for events to become available on any channel, or for shutdown. /// /// Uses `WaitForMultipleObjects` via `spawn_blocking` to avoid blocking the - /// Tokio runtime. The wait array includes all channel signal events plus the - /// shutdown event. + /// Tokio runtime. The wait array puts shutdown first so a stop request wins + /// over any channel that is already signaled. pub fn wait_for_events_blocking(&self, timeout_ms: u32) -> WaitResult { - // Build wait handle array: [channel0_signal, channel1_signal, ..., shutdown_event] - let mut handles: Vec = self.channels.iter().map(|c| c.signal_event).collect(); + // Build wait handle array: [shutdown_event, channel0_signal, channel1_signal, ...] + let mut handles = Vec::with_capacity(self.channels.len() + 1); handles.push(self.shutdown_event); + handles.extend(self.channels.iter().map(|c| c.signal_event)); let result = unsafe { WaitForMultipleObjects(&handles, false, timeout_ms) }; - let shutdown_index = (self.channels.len()) as u32; - match result { r if r == WAIT_TIMEOUT => WaitResult::Timeout, - r if r.0 < WAIT_OBJECT_0.0 + shutdown_index => WaitResult::EventsAvailable, - r if r.0 == WAIT_OBJECT_0.0 + shutdown_index => WaitResult::Shutdown, + r if r == WAIT_OBJECT_0 => WaitResult::Shutdown, + r if r.0 <= WAIT_OBJECT_0.0 + self.channels.len() as u32 => WaitResult::EventsAvailable, _ => { // WAIT_FAILED or unexpected - treat as timeout to avoid tight loop warn!( @@ -459,6 +474,28 @@ impl EventLogSubscription { pub fn pull_events( &mut self, max_events: usize, + ) -> Result, WindowsEventLogError> { + self.pull_events_inner(max_events, true) + } + + /// Pull events for timeout-based speculative recovery. + /// + /// This keeps the same event-drain behavior as `pull_events`, but avoids + /// refreshing per-channel record-count gauges for channels that were empty. + /// Timeout pulls can run repeatedly while the host is idle, so skipping + /// those metadata queries prevents steady `EvtOpenLog`/`EvtGetLogInfo` + /// churn without changing event recovery behavior. + pub fn pull_events_speculative( + &mut self, + max_events: usize, + ) -> Result, WindowsEventLogError> { + self.pull_events_inner(max_events, false) + } + + fn pull_events_inner( + &mut self, + max_events: usize, + update_records_for_empty_channels: bool, ) -> Result, WindowsEventLogError> { let mut all_events = Vec::with_capacity(max_events.min(1000)); let num_channels = self.channels.len().max(1); @@ -479,9 +516,25 @@ impl EventLogSubscription { let mut bookmark_failed = false; let mut channel_count = 0usize; - // Drain loop: keep calling EvtNext until ERROR_NO_MORE_ITEMS or channel budget. - // Only reset the signal once the channel is fully drained; if we hit the - // budget limit the signal stays set so WaitForMultipleObjects returns immediately. + // Reset the signal BEFORE draining to avoid a lost-wakeup race + // (see vectordotdev/vector#25194). The Windows Event Log service + // signals this manual-reset event via SetEvent each time a new + // matching event is recorded; SetEvent on an already-signaled + // event is a no-op, so if we reset AFTER draining, any signal + // that arrives between our last EvtNext and ResetEvent is lost + // — the subscription then hangs until the next event arrives. + // Resetting first means any signal raised during the drain is + // preserved, causing the next WaitForMultipleObjects to return + // immediately. + // + // If we exit the drain loop early (channel budget exhausted or + // bookmark update failed mid-batch), we re-SetEvent at the end + // of this iteration so the next pull_events call revisits this + // channel without waiting for a fresh OS signal. + unsafe { + let _ = ResetEvent(channel_sub.signal_event); + } + 'drain: loop { if channel_count >= channel_limit { break; @@ -501,6 +554,17 @@ impl EventLogSubscription { ) }; + // Test-only hook: lets the lost-wakeup regression test race + // a SetEvent against the drain without thread-timing. No-op + // and zero-cost in non-test builds. + #[cfg(test)] + { + let hook = DRAIN_STEP_HOOK.lock().unwrap().clone(); + if let Some(h) = hook { + h(channel_sub.signal_event); + } + } + if let Err(err) = result { let code = (err.code().0 as u32) & 0xFFFF; if code == ERROR_NO_MORE_ITEMS { @@ -513,6 +577,8 @@ impl EventLogSubscription { channel = %channel_sub.channel ); channel_drained = true; + // Speculative pull on timeout in mod.rs is a safety net if the + // re-subscribed channel does not immediately re-signal. break; } if code == ERROR_EVT_QUERY_RESULT_INVALID_POSITION { @@ -526,7 +592,9 @@ impl EventLogSubscription { message = "Re-subscription succeeded after stale query.", channel = %channel_sub.channel ); - // Retry from fresh subscription — the signal will fire again + // Retry from fresh subscription — the signal will fire again. + // Speculative pull on timeout in mod.rs is a safety net if + // the new subscription does not immediately re-signal. channel_drained = true; break; } @@ -538,10 +606,23 @@ impl EventLogSubscription { ); channel_sub.subscription_active_gauge.set(0.0); channel_drained = true; + // Speculative pull on timeout in mod.rs is a safety net if + // the failed channel does not re-signal after recovery. break; } } } + // Re-arm the signal before returning. We reset it pre-drain + // but are bailing out without confirming the drain completed, + // so if events were left un-drained the next pull_events must + // still revisit this channel without waiting for a fresh OS + // signal. This mirrors the `else` branch below that handles + // budget-exhaustion and bookmark-failure early breaks, and + // avoids the same lost-wakeup symptom (vectordotdev/vector#25194) + // on transient EvtNext failures. + unsafe { + let _ = SetEvent(channel_sub.signal_event); + } return Err(WindowsEventLogError::PullEventsError { channel: channel_sub.channel.clone(), source: err, @@ -697,15 +778,22 @@ impl EventLogSubscription { } if channel_drained && !bookmark_failed { + // Update channel record count gauge for lag detection. + if update_records_for_empty_channels || channel_count > 0 { + super::render::update_channel_records( + &channel_sub.channel, + &channel_sub.channel_records_gauge, + ); + } + } else { + // Drain exited early (budget exhausted or bookmark_failed + // mid-batch). Re-arm the signal so the next pull_events + // revisits this channel immediately without waiting for a + // fresh OS notification. Pairs with the pre-drain ResetEvent + // above. unsafe { - let _ = ResetEvent(channel_sub.signal_event); + let _ = SetEvent(channel_sub.signal_event); } - - // Update channel record count gauge for lag detection. - super::render::update_channel_records( - &channel_sub.channel, - &channel_sub.channel_records_gauge, - ); } } @@ -816,6 +904,15 @@ impl EventLogSubscription { self.shutdown_event.0 } + /// Test-only accessor for the first channel's signal event handle. Used + /// by the lost-wakeup regression test to scope its drain-loop hook to + /// exactly this subscription, so it does not fire on concurrent + /// `pull_events` calls from other tests in the same process. + #[cfg(test)] + pub(super) fn first_channel_signal_raw(&self) -> isize { + self.channels[0].signal_event.0 as isize + } + /// Returns a reference to the rate limiter, if configured. pub const fn rate_limiter( &self, @@ -1005,6 +1102,7 @@ impl Drop for EventLogSubscription { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; async fn create_test_checkpointer() -> (Arc, tempfile::TempDir) { let temp_dir = tempfile::TempDir::new().unwrap(); @@ -1136,6 +1234,31 @@ mod tests { drop(subscription); } + /// Test that shutdown wins when both shutdown and channel handles are signaled. + #[tokio::test] + async fn test_shutdown_signal_takes_priority_over_channel_signal() { + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.event_timeout_ms = 500; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + unsafe { + let handle = HANDLE(subscription.shutdown_event_raw()); + let _ = SetEvent(handle); + } + + let result = subscription.wait_for_events_blocking(0); + assert!( + matches!(result, WaitResult::Shutdown), + "shutdown should take priority over already-signaled channels" + ); + } + /// Test pull_events with read_existing_events=true #[tokio::test] async fn test_pull_events_returns_events() { @@ -1272,4 +1395,209 @@ mod tests { // that the subscription is functional. let _events = subscription.pull_events(100).unwrap_or_default(); } + + /// Proves that `pull_events` works independently of signal state — the + /// invariant the speculative timeout pull in mod.rs relies on. + /// + /// Steps: + /// 1. Subscribe to the Application log with `read_existing_events = true`. + /// 2. Manually clear the channel signal via `ResetEvent`, simulating a lost wakeup. + /// 3. Assert `wait_for_events_blocking` times out (signal cleared, no OS wake-up). + /// 4. Assert `pull_events` still returns events — `EvtNext` fetches from the queue + /// regardless of signal state, so the speculative pull in mod.rs self-heals. + #[tokio::test] + #[serial] + async fn test_pull_events_works_with_cleared_signal() { + // Seed the Application log with a record so the "events remain + // available despite cleared signal" assertion below does not depend + // on whatever backlog the runner happens to have. Freshly provisioned + // CI images can have an empty Application log, which would otherwise + // make `pull_events` legitimately return empty and produce a spurious + // failure unrelated to the invariant under test. + let seed_output = std::process::Command::new("eventcreate") + .args([ + "/T", + "INFORMATION", + "/ID", + "100", + "/L", + "APPLICATION", + "/SO", + "VectorTestSpeculativePullSeed", + "/D", + "seed event for #25194 speculative-pull regression test", + ]) + .output() + .expect("failed to spawn eventcreate — required for deterministic seeding"); + assert!( + seed_output.status.success(), + "eventcreate failed to seed Application log (exit={:?}): stdout={:?} stderr={:?}. \ + This test requires a seeded event to be deterministic; a locked-down runner \ + without the privilege to write to Application cannot run this test reliably.", + seed_output.status.code(), + String::from_utf8_lossy(&seed_output.stdout), + String::from_utf8_lossy(&seed_output.stderr), + ); + // Give the service a moment to persist the record before we subscribe. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.read_existing_events = true; + config.event_timeout_ms = 500; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let mut subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + // Manually clear the signal to simulate a lost wakeup. The seeded + // event above guarantees at least one record is queued in EvtNext + // regardless of the runner's pre-existing log state. + let signal_raw = subscription.first_channel_signal_raw(); + unsafe { + let _ = ResetEvent(HANDLE(signal_raw as *mut std::ffi::c_void)); + } + + // Signal is cleared: an immediate (0ms) poll must report Timeout. + // A 0ms wait reads only the current signal state with no grace + // window, so unrelated Windows system events arriving between the + // `ResetEvent` above and the poll cannot re-signal the handle and + // cause a spurious failure. + let wait_result = subscription.wait_for_events_blocking(0); + + assert!( + matches!(wait_result, WaitResult::Timeout), + "expected Timeout after manual ResetEvent; signal was not cleared" + ); + + // Despite the cleared signal, pull_events must still return events. + // This is the invariant the speculative timeout pull in mod.rs depends on. + let events = subscription.pull_events(100).unwrap_or_default(); + assert!( + !events.is_empty(), + "pull_events must return events independently of signal state; \ + this is the invariant the speculative timeout pull in mod.rs depends on" + ); + } + + /// Regression test for vectordotdev/vector#25194. + /// + /// The Windows Event Log service signals the pull-mode wait handle via + /// `SetEvent` each time a new matching event is recorded. Because the + /// handle is manual-reset, `SetEvent` on an already-signaled handle is + /// a no-op. If `pull_events` resets the signal *after* draining events + /// via `EvtNext`, any signal that fires between the last `EvtNext` and + /// the `ResetEvent` call is silently lost — the subscription then + /// permanently hangs until a subsequent event arrives. + /// + /// The fix is to reset the signal *before* the drain loop, so signals + /// raised during the drain are preserved and the next wait returns + /// immediately. + /// + /// This test pins that invariant by driving the real `pull_events` + /// against a real `EvtSubscribe` handle. It installs a + /// `DRAIN_STEP_HOOK` that runs inside the drain loop after each + /// `EvtNext` and fires `SetEvent` on the subscription's signal + /// handle — simulating the OS signaling a new event arrival during + /// the drain window. After `pull_events` returns, the signal must + /// still be set — observed via a 0ms `wait_for_events_blocking` + /// so the check measures only the reset/preserve behavior of + /// `pull_events` and is not contaminated by unrelated Windows + /// system events arriving during a nonzero wait. Under the old + /// post-drain `ResetEvent` order, the hook's `SetEvent` would be + /// clobbered by the reset and the immediate poll would return + /// `Timeout` — which is exactly what #25194 reports. + #[tokio::test] + #[serial] + async fn test_pull_events_preserves_setevent_during_drain() { + use std::sync::Arc as StdArc; + + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.read_existing_events = true; + config.event_timeout_ms = 1000; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let mut subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + // Capture THIS subscription's signal handle so the hook can scope + // itself to this test. DRAIN_STEP_HOOK is a process-global, and + // cargo runs tests in parallel by default; without handle-keying, + // a concurrent test's pull_events could trigger our one-shot + // hook first, flip `fired`, and SetEvent on the wrong handle. + let target_signal_raw = subscription.first_channel_signal_raw(); + + // Install the drain-loop hook: every EvtNext call inside + // pull_events fires SetEvent on the subscription's signal + // handle. This simulates the OS signaling a fresh event + // mid-drain, which is exactly the race window #25194 exposes. + // The hook only needs to fire once to prove the invariant; we + // use an AtomicBool to keep it deterministic. The hook is keyed + // to `target_signal_raw` so concurrent pull_events calls from + // other tests no-op here. + let fired = StdArc::new(std::sync::atomic::AtomicBool::new(false)); + { + let fired = StdArc::clone(&fired); + let hook: StdArc = StdArc::new(move |signal: HANDLE| { + if signal.0 as isize != target_signal_raw { + return; + } + if !fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + unsafe { + let _ = SetEvent(signal); + } + } + }); + *DRAIN_STEP_HOOK.lock().unwrap() = Some(hook); + } + + // Drop-guard: clear the hook even if the test panics, so it + // doesn't contaminate other tests in the same process. + struct HookGuard; + impl Drop for HookGuard { + fn drop(&mut self) { + *DRAIN_STEP_HOOK.lock().unwrap() = None; + } + } + let _guard = HookGuard; + + // Drive pull_events with a very large budget so the drain + // exits via ERROR_NO_MORE_ITEMS (channel_drained = true), + // which is the path that ran the post-drain ResetEvent in the + // old buggy code. Exiting via budget exhaustion would skip + // that reset and cause this test to false-pass against the + // pre-fix code. + let _ = subscription.pull_events(usize::MAX).unwrap_or_default(); + + assert!( + fired.load(std::sync::atomic::Ordering::SeqCst), + "drain-loop hook never ran — pull_events must call EvtNext \ + at least once even on an empty channel" + ); + + // Observe the signal state IMMEDIATELY with a 0ms wait. We want + // to know whether pull_events's reset clobbered the hook's + // SetEvent — NOT whether new real events arrive during some + // wait window. A nonzero timeout against the live Application + // channel lets arbitrary Windows system events re-signal us + // and false-pass against the pre-fix code. 0ms = WaitForMultiple- + // Objects returns the current state with no grace period, so + // only the reset/preserve behavior of pull_events is measured. + let result = subscription.wait_for_events_blocking(0); + + match result { + WaitResult::EventsAvailable => {} + WaitResult::Timeout => panic!( + "signal set during the drain window was lost — this is the \ + lost-wakeup race from vectordotdev/vector#25194. \ + pull_events must call ResetEvent BEFORE draining, not after." + ), + WaitResult::Shutdown => panic!("unexpected shutdown"), + } + } } From 59a53b138d127fdca68d260628d1dc0035b3f711 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 1 May 2026 13:43:39 -0400 Subject: [PATCH 137/364] fix(internal docs): remove type-level default on StatusCode (#25345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(vector-config): remove type-level default on StatusCode The Configurable impl for http::StatusCode set a type-level default of 200, which the CUE generator propagated into the `uint` schema for each item of `Vec` — including RetryStrategy::Custom.status_codes introduced in #25057. The CUE schema for #TypeArray hardcodes items.type._args.required = true, which forbids `default` on item primitives, so cue-build failed on every http-based sink with: components.sinks..configuration.retry_strategy.type.object: field not allowed Drop the type-level default. The only field that needs a default for StatusCode is sources::http_server::response_code, which already sets its own via #[serde(default = "default_http_response_code")]. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/vector-config/src/http.rs | 1 - website/cue/reference/components/sinks/generated/appsignal.cue | 2 +- website/cue/reference/components/sinks/generated/axiom.cue | 2 +- .../components/sinks/generated/azure_logs_ingestion.cue | 2 +- .../reference/components/sinks/generated/azure_monitor_logs.cue | 2 +- .../cue/reference/components/sinks/generated/datadog_events.cue | 2 +- .../components/sinks/generated/gcp_stackdriver_logs.cue | 2 +- .../components/sinks/generated/gcp_stackdriver_metrics.cue | 2 +- website/cue/reference/components/sinks/generated/honeycomb.cue | 2 +- website/cue/reference/components/sinks/generated/http.cue | 2 +- website/cue/reference/components/sinks/generated/keep.cue | 2 +- .../cue/reference/components/sinks/generated/opentelemetry.cue | 2 +- .../components/sinks/generated/prometheus_remote_write.cue | 2 +- 13 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/vector-config/src/http.rs b/lib/vector-config/src/http.rs index da51ea1cc156e..1f61630bf7cbd 100644 --- a/lib/vector-config/src/http.rs +++ b/lib/vector-config/src/http.rs @@ -28,7 +28,6 @@ impl Configurable for StatusCode { fn metadata() -> Metadata { let mut metadata = Metadata::default(); metadata.set_description("HTTP response status code"); - metadata.set_default_value(StatusCode::OK); metadata.add_custom_attribute(CustomAttribute::kv( constants::DOCS_META_NUMERIC_TYPE, NumberClass::Unsigned, diff --git a/website/cue/reference/components/sinks/generated/appsignal.cue b/website/cue/reference/components/sinks/generated/appsignal.cue index 47943be3c1f93..41769e4ec2a41 100644 --- a/website/cue/reference/components/sinks/generated/appsignal.cue +++ b/website/cue/reference/components/sinks/generated/appsignal.cue @@ -337,7 +337,7 @@ generated: components: sinks: appsignal: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/axiom.cue b/website/cue/reference/components/sinks/generated/axiom.cue index a1f42404ac0c1..e08085b79926b 100644 --- a/website/cue/reference/components/sinks/generated/axiom.cue +++ b/website/cue/reference/components/sinks/generated/axiom.cue @@ -319,7 +319,7 @@ generated: components: sinks: axiom: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 68944f5a6e7d5..9300dd205df64 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -422,7 +422,7 @@ generated: components: sinks: azure_logs_ingestion: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue index c7039b70097e1..205b671072bdb 100644 --- a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue @@ -328,7 +328,7 @@ generated: components: sinks: azure_monitor_logs: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/datadog_events.cue b/website/cue/reference/components/sinks/generated/datadog_events.cue index 23b9fb43d6413..f823c05445a2e 100644 --- a/website/cue/reference/components/sinks/generated/datadog_events.cue +++ b/website/cue/reference/components/sinks/generated/datadog_events.cue @@ -256,7 +256,7 @@ generated: components: sinks: datadog_events: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue index 9d5e76262bfd2..2685eda9ffb10 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue @@ -434,7 +434,7 @@ generated: components: sinks: gcp_stackdriver_logs: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue index 52c344e278250..d9897bbaf092e 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue @@ -348,7 +348,7 @@ generated: components: sinks: gcp_stackdriver_metrics: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/honeycomb.cue b/website/cue/reference/components/sinks/generated/honeycomb.cue index c65064a135a00..5c3acaed97ffa 100644 --- a/website/cue/reference/components/sinks/generated/honeycomb.cue +++ b/website/cue/reference/components/sinks/generated/honeycomb.cue @@ -335,7 +335,7 @@ generated: components: sinks: honeycomb: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/http.cue b/website/cue/reference/components/sinks/generated/http.cue index 8523a27dd0d9b..e08fd535971e6 100644 --- a/website/cue/reference/components/sinks/generated/http.cue +++ b/website/cue/reference/components/sinks/generated/http.cue @@ -1038,7 +1038,7 @@ generated: components: sinks: http: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/keep.cue b/website/cue/reference/components/sinks/generated/keep.cue index d2ed08114a2a9..43b8500746291 100644 --- a/website/cue/reference/components/sinks/generated/keep.cue +++ b/website/cue/reference/components/sinks/generated/keep.cue @@ -300,7 +300,7 @@ generated: components: sinks: keep: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 4e8e9d9ff7210..3d52d15690bed 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1039,7 +1039,7 @@ generated: components: sinks: opentelemetry: configuration: protocol: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue index 5d74d52359bdd..0156777c09fb2 100644 --- a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue @@ -553,7 +553,7 @@ generated: components: sinks: prometheus_remote_write: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." From f1b2c3a3f14f59ab9615829852da6e65a7d2c512 Mon Sep 17 00:00:00 2001 From: tot19 <31141271+tot19@users.noreply.github.com> Date: Sat, 2 May 2026 02:06:42 +0900 Subject: [PATCH 138/364] fix(sources): add windows_event_log source metadata (#25337) * fix(windows_event_log): add standard source metadata * chore(changelog): add windows_event_log metadata fix note --- ...332_windows_event_log_source_metadata.fix.md | 3 +++ src/sources/windows_event_log/mod.rs | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog.d/25332_windows_event_log_source_metadata.fix.md diff --git a/changelog.d/25332_windows_event_log_source_metadata.fix.md b/changelog.d/25332_windows_event_log_source_metadata.fix.md new file mode 100644 index 0000000000000..936e773cc8588 --- /dev/null +++ b/changelog.d/25332_windows_event_log_source_metadata.fix.md @@ -0,0 +1,3 @@ +The `windows_event_log` source now adds standard source metadata, including `source_type`, to emitted log events. + +authors: tot19 diff --git a/src/sources/windows_event_log/mod.rs b/src/sources/windows_event_log/mod.rs index 28dfc1db4cb79..9fb2cff142ee6 100644 --- a/src/sources/windows_event_log/mod.rs +++ b/src/sources/windows_event_log/mod.rs @@ -25,6 +25,7 @@ cfg_if::cfg_if! { use std::path::PathBuf; use std::sync::Arc; + use chrono::Utc; use futures::StreamExt; use vector_lib::EstimatedJsonEncodedSizeOf; use vector_lib::finalizer::OrderedFinalizer; @@ -166,6 +167,7 @@ impl Finalizer { async fn process_event_batch( events: Vec, parser: &EventLogParser, + log_namespace: LogNamespace, acknowledgements: bool, subscription: &EventLogSubscription, out: &mut SourceSender, @@ -189,6 +191,12 @@ async fn process_event_batch( let event_id = event.event_id; match parser.parse_event(event) { Ok(mut log_event) => { + log_namespace.insert_standard_vector_source_metadata( + &mut log_event, + WindowsEventLogConfig::NAME, + Utc::now(), + ); + let byte_size = log_event.estimated_json_encoded_size_of(); total_byte_size += byte_size.get(); if let Some(ref batch) = batch { @@ -416,6 +424,7 @@ impl WindowsEventLogSource { if process_event_batch( events, &parser, + self.log_namespace, acknowledgements, &subscription, &mut out, @@ -516,6 +525,7 @@ impl WindowsEventLogSource { if process_event_batch( events, &parser, + self.log_namespace, acknowledgements, &subscription, &mut out, @@ -664,8 +674,11 @@ impl SourceConfig for WindowsEventLogConfig { ), ])), [LogNamespace::Vector], - ), - LogNamespace::Legacy => vector_lib::schema::Definition::any(), + ) + .with_standard_vector_source_metadata(), + LogNamespace::Legacy => { + vector_lib::schema::Definition::any().with_standard_vector_source_metadata() + } }; vec![SourceOutput::new_maybe_logs( From 9f15e23943d4347a6f2171eaa97a921a5e58d457 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 1 May 2026 13:56:58 -0400 Subject: [PATCH 139/364] fix(ci): bump cue and add cue-build step to Check Cue docs (#25346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(vector-config): remove type-level default on StatusCode The Configurable impl for http::StatusCode set a type-level default of 200, which the CUE generator propagated into the `uint` schema for each item of `Vec` — including RetryStrategy::Custom.status_codes introduced in #25057. The CUE schema for #TypeArray hardcodes items.type._args.required = true, which forbids `default` on item primitives, so cue-build failed on every http-based sink with: components.sinks..configuration.retry_strategy.type.object: field not allowed Drop the type-level default. The only field that needs a default for StatusCode is sources::http_server::response_code, which already sets its own via #[serde(default = "default_http_response_code")]. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): bump cue from v0.10.0 to v0.16.1 CI's cue v0.10.0 was lax enough that schema violations like the "field not allowed" error fixed in the previous commit slipped through the existing `Check Cue docs` step. Newer cue versions enforce closed struct constraints in disjunctions strictly, matching what local developers see when running the website build. Co-Authored-By: Claude Opus 4.7 (1M context) * ci(test): also run make cue-build in Check Cue docs The Check Cue docs job previously only ran make check-docs, which runs cue vet. Adding make cue-build (which runs cue export and materializes website/data/docs.json) catches schema-unification failures that the website build would hit and that vet may miss. The new step reuses the cue + VRL docs already produced by the prior make check-docs invocation, so it adds only the cost of one cue export. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): bump cue to v0.16.1 in remaining installers Keep all three cue installers in lockstep with .github/actions/setup to avoid version skew that would let developer / website-Dockerfile paths accept cue sources the CI Check Cue docs job rejects. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 6 +++--- .github/workflows/test.yml | 2 ++ scripts/environment/bootstrap-ubuntu-24.04.sh | 6 +++--- website/Dockerfile | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6faa185204ced..3c7e645e6831b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -237,10 +237,10 @@ runs: echo "Installing cue" TEMP=$(mktemp -d) curl \ - -L https://github.com/cue-lang/cue/releases/download/v0.10.0/cue_v0.10.0_linux_amd64.tar.gz \ - -o "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" + -L https://github.com/cue-lang/cue/releases/download/v0.16.1/cue_v0.16.1_linux_amd64.tar.gz \ + -o "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" tar \ - -xvf "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" \ + -xvf "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" \ -C "${TEMP}" sudo cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e5c28aec476c..5244d8110d823 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,6 +121,8 @@ jobs: rust: true cue: true - run: make check-docs + - name: Check Cue website build + run: cd website && make cue-build check-markdown: name: Check Markdown diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh index 9d135f779b933..4298fb4dba94b 100755 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -56,10 +56,10 @@ apt-get install --yes \ # Cue TEMP=$(mktemp -d) curl \ - -L https://github.com/cue-lang/cue/releases/download/v0.10.0/cue_v0.10.0_linux_amd64.tar.gz \ - -o "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" + -L https://github.com/cue-lang/cue/releases/download/v0.16.1/cue_v0.16.1_linux_amd64.tar.gz \ + -o "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" tar \ - -xvf "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" \ + -xvf "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" \ -C "${TEMP}" cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" diff --git a/website/Dockerfile b/website/Dockerfile index e9946941bb459..d98c0f2103ab1 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim ARG HUGO_VERSION=0.154.5 -ARG CUE_VERSION=0.10.0 +ARG CUE_VERSION=0.16.1 ARG NODE_VERSION=24 RUN apt-get update && apt-get install -y --no-install-recommends \ From 7923556313d66be69e638022e10fe3fd13f468ac Mon Sep 17 00:00:00 2001 From: Flavio Cruz <87035001+flaviofcruz@users.noreply.github.com> Date: Fri, 1 May 2026 14:57:31 -0400 Subject: [PATCH 140/364] fix(clickhouse sink, aws_s3 sink): use dedicated batch_encoding types (#25340) * Fix the clickhouse sink so that it only accepts arrow * Do not allow arrow stream or parquet on AWS S3 * reformat * Add changelog fragment * Remove stale comments * chore(aws_s3 sink): exhaustive match for batch_encoding default extension So adding a future S3BatchEncoding variant is a compile error rather than silently defaulting to "parquet". Co-Authored-By: Claude Opus 4.7 (1M context) * chore(aws_s3 sink): rename shadowing parquet_config bind in tests Avoid rebinding parquet_config to a borrow of itself from config.batch_encoding; use a short p binding instead. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(changelog): rewrite batch_encoding fragment for users Drop internal type names and dev jargon; lead with the user-visible behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) * test(aws_s3 sink, clickhouse sink): reject unsupported batch_encoding codec at parse time Add per-sink deserialization-failure tests that pin the schema-tightening behavior introduced by the dedicated wrapper enums: - aws_s3 rejects codec: arrow_stream - clickhouse rejects codec: parquet Previously both codecs were accepted by serde and rejected later at sink-build time; the new wrapper enums move rejection up to parse time, and these tests prevent silent regression of that contract. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(aws_s3 sink): re-export S3BatchEncoding from aws_s3 module Programmatic users constructing S3SinkConfig directly need to be able to set the batch_encoding field. With config kept private in src/sinks/aws_s3/mod.rs, S3BatchEncoding was unnameable outside the crate, regressing callers that previously used BatchSerializerConfig::Parquet(...) at the same field. Gated by codecs-parquet to mirror the type and field. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.7 (1M context) --- ...use_aws_s3_dedicated_batch_encoding.fix.md | 3 + src/sinks/aws_s3/config.rs | 139 ++++++++---------- src/sinks/aws_s3/integration_tests.rs | 6 +- src/sinks/aws_s3/mod.rs | 2 + src/sinks/clickhouse/config.rs | 61 ++++++-- src/sinks/clickhouse/integration_tests.rs | 14 +- .../components/sinks/generated/aws_s3.cue | 54 ++----- .../components/sinks/generated/clickhouse.cue | 80 ++-------- 8 files changed, 147 insertions(+), 212 deletions(-) create mode 100644 changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md diff --git a/changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md b/changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md new file mode 100644 index 0000000000000..f39347d4c47bd --- /dev/null +++ b/changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md @@ -0,0 +1,3 @@ +The `aws_s3` and `clickhouse` sinks now correctly advertise only the `batch_encoding.codec` values they actually support: `parquet` for `aws_s3` and `arrow_stream` for `clickhouse`. Previously the documentation and configuration schema listed both codecs for both sinks, even though picking the wrong one produced a startup error. + +authors: flaviofcruz diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 6e8d2e248c518..05055e6ea0167 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -3,7 +3,7 @@ use tower::ServiceBuilder; #[cfg(feature = "codecs-parquet")] use vector_lib::codecs::BatchEncoder; #[cfg(feature = "codecs-parquet")] -use vector_lib::codecs::encoding::BatchSerializerConfig; +use vector_lib::codecs::encoding::{BatchSerializerConfig, format::ParquetSerializerConfig}; use vector_lib::{ TimeZone, codecs::{ @@ -37,6 +37,21 @@ use crate::{ tls::TlsConfig, }; +/// Batch encoding configuration for the `aws_s3` sink. +#[cfg(feature = "codecs-parquet")] +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum S3BatchEncoding { + /// Encodes events in [Apache Parquet][apache_parquet] columnar format. + /// + /// [apache_parquet]: https://parquet.apache.org/ + Parquet(ParquetSerializerConfig), +} + /// Configuration for the `aws_s3` sink. #[configurable_component(sink( "aws_s3", @@ -111,15 +126,13 @@ pub struct S3SinkConfig { /// Batch encoding configuration for columnar formats. /// - /// When set, events are encoded together as a batch in a columnar format (for example, Parquet) + /// When set, events are encoded together as a batch in a columnar format (Parquet) /// instead of the standard per-event framing-based encoding. The columnar format handles /// its own internal compression, so the top-level `compression` setting is bypassed. - /// - /// Only the `parquet` codec is supported by the AWS S3 sink. #[cfg(feature = "codecs-parquet")] #[configurable(derived)] #[serde(default)] - pub batch_encoding: Option, + pub batch_encoding: Option, /// Compression configuration. /// @@ -220,8 +233,10 @@ impl SinkConfig for S3SinkConfig { fn input(&self) -> Input { #[cfg(feature = "codecs-parquet")] - if let Some(batch_config) = &self.batch_encoding { - return Input::new(batch_config.input_type()); + if let Some(batch_encoding) = &self.batch_encoding { + let S3BatchEncoding::Parquet(parquet_config) = batch_encoding; + let resolved = BatchSerializerConfig::Parquet(parquet_config.clone()); + return Input::new(resolved.input_type()); } Input::new(self.encoding.config().1.input_type()) } @@ -272,15 +287,11 @@ impl S3SinkConfig { // When batch_encoding is configured (e.g., Parquet), use batch mode // with internal compression and appropriate file extension. #[cfg(feature = "codecs-parquet")] - if let Some(batch_config) = &self.batch_encoding { - if !matches!(batch_config, BatchSerializerConfig::Parquet(_)) { - return Err( - "batch_encoding only supports encoding with parquet format for amazon s3 sink" - .into(), - ); - } + if let Some(batch_encoding) = &self.batch_encoding { + let S3BatchEncoding::Parquet(parquet_config) = batch_encoding; + let resolved_batch_config = BatchSerializerConfig::Parquet(parquet_config.clone()); - let batch_serializer = batch_config.build_batch_serializer()?; + let batch_serializer = resolved_batch_config.build_batch_serializer()?; let batch_encoder = BatchEncoder::new(batch_serializer); // Auto-detect Content-Type from batch format. Users can still @@ -292,15 +303,14 @@ impl S3SinkConfig { let encoder = EncoderKind::Batch(batch_encoder); - // Auto-detect file extension from batch format - let filename_extension = - self.filename_extension - .clone() - .or_else(|| match batch_config { - BatchSerializerConfig::Parquet(_) => Some("parquet".to_string()), - #[allow(unreachable_patterns)] - _ => None, - }); + let filename_extension = self.filename_extension.clone().or_else(|| { + Some( + match batch_encoding { + S3BatchEncoding::Parquet(_) => "parquet", + } + .to_string(), + ) + }); if self.compression != Compression::None { warn!("Top level compression setting ignored when batch_encoding set to parquet.") @@ -392,15 +402,10 @@ mod tests { let batch_enc = config .batch_encoding .expect("batch_encoding should be Some"); - match batch_enc { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(ref p) => { - use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; - assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); - assert_eq!(p.compression, ParquetCompression::Snappy); - } - #[allow(unreachable_patterns)] - _ => panic!("expected Parquet variant"), - } + let super::S3BatchEncoding::Parquet(ref p) = batch_enc; + use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; + assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); + assert_eq!(p.compression, ParquetCompression::Snappy); } /// Content-Type must be auto-detected as `application/vnd.apache.parquet` @@ -432,7 +437,7 @@ mod tests { options: S3Options::default(), region: crate::aws::RegionOrEndpoint::with_both("us-east-1", "http://localhost:4566"), encoding: (None::, TextSerializerConfig::default()).into(), - batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + batch_encoding: Some(super::S3BatchEncoding::Parquet(parquet_config)), compression: Compression::None, batch: BatchConfig::::default(), request: Default::default(), @@ -444,7 +449,8 @@ mod tests { retry_strategy: Default::default(), }; - let batch_config = config.batch_encoding.as_ref().unwrap(); + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.as_ref().unwrap(); + let batch_config = BatchSerializerConfig::Parquet(p.clone()); let batch_serializer = batch_config.build_batch_serializer().unwrap(); let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); @@ -484,7 +490,8 @@ mod tests { ) .unwrap(); - let batch_config = config.batch_encoding.as_ref().unwrap(); + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.as_ref().unwrap(); + let batch_config = vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p.clone()); let batch_serializer = batch_config.build_batch_serializer().unwrap(); let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); @@ -500,43 +507,27 @@ mod tests { ); } - /// Parquet filename extension defaults to `.parquet` when not explicitly set. + /// Codecs other than `parquet` must be rejected at parse time, since + /// `S3BatchEncoding` only exposes the `parquet` variant. #[cfg(feature = "codecs-parquet")] #[test] - fn parquet_filename_extension_defaults_to_parquet() { - let config: S3SinkConfig = toml::from_str( + fn parquet_batch_encoding_rejects_unsupported_codec() { + let err = serde_yaml::from_str::( r#" - bucket = "test-bucket" - compression = "none" - - [encoding] - codec = "text" - - [batch_encoding] - codec = "parquet" - schema_mode = "auto_infer" + bucket: test-bucket + compression: none + encoding: + codec: text + batch_encoding: + codec: arrow_stream "#, ) - .unwrap(); + .unwrap_err(); assert!( - config.filename_extension.is_none(), - "fixture must not set filename_extension" + err.to_string().contains("arrow_stream"), + "expected error to mention the offending codec, got: {err}" ); - - let batch_config = config.batch_encoding.as_ref().unwrap(); - let extension = config - .filename_extension - .clone() - .or_else(|| match batch_config { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(_) => { - Some("parquet".to_string()) - } - #[allow(unreachable_patterns)] - _ => None, - }); - - assert_eq!(extension.as_deref(), Some("parquet")); } /// Explicit filename_extension overrides the `.parquet` default. @@ -582,13 +573,8 @@ mod tests { ) .unwrap(); - match config.batch_encoding.unwrap() { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { - assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); - } - #[allow(unreachable_patterns)] - _ => panic!("expected Parquet variant"), - } + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.unwrap(); + assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); } /// Explicit `schema_mode = "strict"` is correctly parsed. @@ -613,12 +599,7 @@ mod tests { ) .unwrap(); - match config.batch_encoding.unwrap() { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { - assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); - } - #[allow(unreachable_patterns)] - _ => panic!("expected Parquet variant"), - } + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.unwrap(); + assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); } } diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index a1a2c10af2201..3a7e156208387 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -7,6 +7,8 @@ use std::{ time::Duration, }; +#[cfg(feature = "codecs-parquet")] +use super::config::S3BatchEncoding; use aws_sdk_s3::{ Client as S3Client, operation::{create_bucket::CreateBucketError, get_object::GetObjectOutput}, @@ -22,8 +24,6 @@ use futures::{Stream, stream}; use similar_asserts::assert_eq; use tempfile::TempDir; use tokio_stream::StreamExt; -#[cfg(feature = "codecs-parquet")] -use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ buffers::{BufferConfig, BufferType, WhenFull}, codecs::{TextSerializerConfig, encoding::FramingConfig}, @@ -502,7 +502,7 @@ async fn s3_parquet_insert_message() { }; let config = S3SinkConfig { - batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + batch_encoding: Some(S3BatchEncoding::Parquet(parquet_config)), ..config(&bucket, 100, 5.0) }; diff --git a/src/sinks/aws_s3/mod.rs b/src/sinks/aws_s3/mod.rs index 3027de1bb6287..4589746175558 100644 --- a/src/sinks/aws_s3/mod.rs +++ b/src/sinks/aws_s3/mod.rs @@ -3,4 +3,6 @@ mod sink; mod integration_tests; +#[cfg(feature = "codecs-parquet")] +pub use config::S3BatchEncoding; pub use config::S3SinkConfig; diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 435b2504e6a9e..53050dca831ea 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -4,8 +4,8 @@ use std::fmt; use http::{Request, StatusCode, Uri}; use hyper::Body; +use vector_lib::codecs::encoding::ArrowStreamSerializerConfig; use vector_lib::codecs::encoding::format::SchemaProvider; -use vector_lib::codecs::encoding::{ArrowStreamSerializerConfig, BatchSerializerConfig}; use super::{ request_builder::ClickhouseRequestBuilder, @@ -45,6 +45,23 @@ pub enum Format { ArrowStream, } +/// Batch encoding configuration for the `clickhouse` sink. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum ClickhouseBatchEncoding { + /// Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + /// + /// This is the streaming variant of the Arrow IPC format, which writes + /// a continuous stream of record batches. + /// + /// [apache_arrow]: https://arrow.apache.org/ + ArrowStream(ArrowStreamSerializerConfig), +} + impl fmt::Display for Format { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -104,11 +121,9 @@ pub struct ClickhouseConfig { /// /// When specified, events are encoded together as a single batch. /// This is mutually exclusive with per-event encoding based on the `format` field. - /// - /// Only the `arrow_stream` codec is supported by the ClickHouse sink. #[configurable(derived)] #[serde(default)] - pub batch_encoding: Option, + pub batch_encoding: Option, #[configurable(derived)] #[serde(default)] @@ -282,6 +297,7 @@ impl ClickhouseConfig { if let Some(batch_encoding) = &self.batch_encoding { use vector_lib::codecs::BatchEncoder; + use vector_lib::codecs::encoding::BatchSerializerConfig; // Validate that batch_encoding is only compatible with ArrowStream format if self.format != Format::ArrowStream { @@ -292,16 +308,8 @@ impl ClickhouseConfig { .into()); } - let mut arrow_config = match batch_encoding { - BatchSerializerConfig::ArrowStream(config) => config.clone(), - #[cfg(feature = "codecs-parquet")] - BatchSerializerConfig::Parquet(_) => { - return Err( - "ClickHouse sink does not support Parquet batch encoding. Use 'arrow_stream' instead." - .into(), - ); - } - }; + let ClickhouseBatchEncoding::ArrowStream(arrow_config) = batch_encoding; + let mut arrow_config = arrow_config.clone(); self.resolve_arrow_schema( client, @@ -433,10 +441,31 @@ mod tests { ); } + /// Codecs other than `arrow_stream` must be rejected at parse time, since + /// `ClickhouseBatchEncoding` only exposes the `arrow_stream` variant. + #[cfg(feature = "codecs-parquet")] + #[test] + fn batch_encoding_rejects_unsupported_codec() { + let err = serde_yaml::from_str::( + r#" + endpoint: http://localhost:8123 + table: test_table + batch_encoding: + codec: parquet + "#, + ) + .unwrap_err(); + + assert!( + err.to_string().contains("parquet"), + "expected error to mention the offending codec, got: {err}" + ); + } + /// Helper to create a minimal ClickhouseConfig for testing fn create_test_config( format: Format, - batch_encoding: Option, + batch_encoding: Option, ) -> ClickhouseConfig { ClickhouseConfig { endpoint: "http://localhost:8123".parse::().unwrap().into(), @@ -469,7 +498,7 @@ mod tests { for (format, format_name) in incompatible_formats { let config = create_test_config( format, - Some(BatchSerializerConfig::ArrowStream( + Some(ClickhouseBatchEncoding::ArrowStream( ArrowStreamSerializerConfig::default(), )), ); diff --git a/src/sinks/clickhouse/integration_tests.rs b/src/sinks/clickhouse/integration_tests.rs index a32771bf55730..2e27a20340dfa 100644 --- a/src/sinks/clickhouse/integration_tests.rs +++ b/src/sinks/clickhouse/integration_tests.rs @@ -18,7 +18,7 @@ use serde::Deserialize; use serde_json::Value; use tokio::time::{Duration, timeout}; use vector_lib::{ - codecs::encoding::{ArrowStreamSerializerConfig, BatchSerializerConfig}, + codecs::encoding::ArrowStreamSerializerConfig, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, LogEvent}, lookup::PathPrefix, }; @@ -28,7 +28,7 @@ use crate::{ codecs::{TimestampFormat, Transformer}, config::{SinkConfig, SinkContext, log_schema}, sinks::{ - clickhouse::config::ClickhouseConfig, + clickhouse::config::{ClickhouseBatchEncoding, ClickhouseConfig}, util::{BatchConfig, Compression, TowerRequestConfig}, }, test_util::{ @@ -502,7 +502,7 @@ async fn insert_events_arrow_format() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -574,7 +574,7 @@ async fn insert_events_arrow_with_schema_fetching() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -657,7 +657,7 @@ async fn test_complex_types() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(arrow_config)), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(arrow_config)), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -1231,7 +1231,7 @@ async fn test_missing_required_field_emits_null_constraint_error() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -1323,7 +1323,7 @@ async fn arrow_schema_excludes_non_insertable_columns() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream( + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream( ArrowStreamSerializerConfig::default(), )), batch, diff --git a/website/cue/reference/components/sinks/generated/aws_s3.cue b/website/cue/reference/components/sinks/generated/aws_s3.cue index 1a4713e2897dd..5627d4636698a 100644 --- a/website/cue/reference/components/sinks/generated/aws_s3.cue +++ b/website/cue/reference/components/sinks/generated/aws_s3.cue @@ -260,52 +260,28 @@ generated: components: sinks: aws_s3: configuration: { description: """ Batch encoding configuration for columnar formats. - When set, events are encoded together as a batch in a columnar format (for example, Parquet) + When set, events are encoded together as a batch in a columnar format (Parquet) instead of the standard per-event framing-based encoding. The columnar format handles its own internal compression, so the top-level `compression` setting is bypassed. - - Only the `parquet` codec is supported by the AWS S3 sink. """ required: false type: object: options: { - allow_nullable_fields: { + codec: { description: """ - Allow null values for non-nullable fields in the schema. - - When enabled, missing or incompatible values are encoded as null, even for fields - marked as non-nullable in the Arrow schema. This is useful when working with downstream - systems that can handle null values through defaults, computed columns, or other mechanisms. + Encodes events in [Apache Parquet][apache_parquet] columnar format. - When disabled (default), missing values for non-nullable fields results in encoding errors. This is to - help ensure all required data is present before sending it to the sink. + [apache_parquet]: https://parquet.apache.org/ """ - relevant_when: "codec = \"arrow_stream\"" - required: false - type: bool: default: false - } - codec: { - description: "The codec to use for batch encoding events." - required: true - type: string: enum: { - arrow_stream: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. - - [apache_arrow]: https://arrow.apache.org/ - """ - parquet: """ - Encodes events in [Apache Parquet][apache_parquet] columnar format. + required: true + type: string: enum: parquet: """ + Encodes events in [Apache Parquet][apache_parquet] columnar format. - [apache_parquet]: https://parquet.apache.org/ - """ - } + [apache_parquet]: https://parquet.apache.org/ + """ } compression: { - description: "Compression codec applied per column page inside the Parquet file." - relevant_when: "codec = \"parquet\"" - required: false + description: "Compression codec applied per column page inside the Parquet file." + required: false type: object: options: { algorithm: { description: "Compression codec applied per column page inside the Parquet file." @@ -336,14 +312,12 @@ generated: components: sinks: aws_s3: configuration: { Required unless `schema_mode` is `auto_infer`. The file must contain a valid Parquet message type definition. """ - relevant_when: "codec = \"parquet\"" - required: false + required: false type: string: {} } schema_mode: { - description: "Controls how events with fields not present in the schema are handled." - relevant_when: "codec = \"parquet\"" - required: false + description: "Controls how events with fields not present in the schema are handled." + required: false type: string: { default: "relaxed" enum: { diff --git a/website/cue/reference/components/sinks/generated/clickhouse.cue b/website/cue/reference/components/sinks/generated/clickhouse.cue index 547fbc943e9d9..e07014f38a308 100644 --- a/website/cue/reference/components/sinks/generated/clickhouse.cue +++ b/website/cue/reference/components/sinks/generated/clickhouse.cue @@ -249,8 +249,6 @@ generated: components: sinks: clickhouse: configuration: { When specified, events are encoded together as a single batch. This is mutually exclusive with per-event encoding based on the `format` field. - - Only the `arrow_stream` codec is supported by the ClickHouse sink. """ required: false type: object: options: { @@ -265,79 +263,27 @@ generated: components: sinks: clickhouse: configuration: { When disabled (default), missing values for non-nullable fields results in encoding errors. This is to help ensure all required data is present before sending it to the sink. """ - relevant_when: "codec = \"arrow_stream\"" - required: false + required: false type: bool: default: false } codec: { - description: "The codec to use for batch encoding events." - required: true - type: string: enum: { - arrow_stream: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + description: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. - [apache_arrow]: https://arrow.apache.org/ - """ - parquet: """ - Encodes events in [Apache Parquet][apache_parquet] columnar format. + [apache_arrow]: https://arrow.apache.org/ + """ + required: true + type: string: enum: arrow_stream: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - [apache_parquet]: https://parquet.apache.org/ - """ - } - } - compression: { - description: "Compression codec applied per column page inside the Parquet file." - relevant_when: "codec = \"parquet\"" - required: false - type: object: options: { - algorithm: { - description: "Compression codec applied per column page inside the Parquet file." - required: false - type: string: { - default: "snappy" - enum: { - gzip: "Gzip compression. Level must be between 1 and 9." - lz4: "LZ4 raw compression" - none: "No compression" - snappy: "Snappy compression (no level)." - zstd: "Zstd compression. Level must be between 1 and 21." - } - } - } - level: { - description: "Compression level (1–21). This is the range Vector supports; higher values compress more but are slower." - relevant_when: "algorithm = \"zstd\" or algorithm = \"gzip\"" - required: true - type: uint: {} - } - } - } - schema_file: { - description: """ - Path to a native Parquet schema file (`.schema`). + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. - Required unless `schema_mode` is `auto_infer`. The file must contain a valid - Parquet message type definition. + [apache_arrow]: https://arrow.apache.org/ """ - relevant_when: "codec = \"parquet\"" - required: false - type: string: {} - } - schema_mode: { - description: "Controls how events with fields not present in the schema are handled." - relevant_when: "codec = \"parquet\"" - required: false - type: string: { - default: "relaxed" - enum: { - auto_infer: "Auto infer schema based on the batch. No schema file needed." - relaxed: "Missing fields become null. Extra fields are silently dropped." - strict: "Missing fields become null. Extra fields cause an error." - } - } } } } From e1c6139b9717f36027b0ac9fe4d20276da4da128 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 2 May 2026 16:51:00 -0400 Subject: [PATCH 141/364] chore(metrics): introduce enums for metric names (#25342) * Add MetricName into vector-common * Use EventName in lib/ * Use MetricName in src/internal_events * Format * Fix EventName imports * Use MetricName in k8s * Use MetricName in aws_sqs * Use EventName everywhere and make clippy pass * Separate MetricName -> CounterName/GaugeName/HistogramName * Merge imports * Finish MetricName separation * chore: update check-events script to support typed metric name enums * chore: fix stale MetricName doc references in macro wrappers * chore: remove stale doc paragraph from counter! macro * Fix clippy.toml * Use first metric names instead of made up names to remove clippy::disallowed_macros * chore: replace MetricName with CounterName in Windows internal events --- Cargo.lock | 3 + Cargo.toml | 1 + benches/metrics_snapshot.rs | 6 +- clippy.toml | 6 + lib/codecs/src/internal_events.rs | 29 +- lib/vector-buffers/src/internal_events.rs | 53 +-- .../src/topology/channel/limited_queue.rs | 1 + lib/vector-common/Cargo.toml | 1 + .../src/internal_event/bytes_received.rs | 8 +- .../src/internal_event/bytes_sent.rs | 8 +- .../src/internal_event/cached_event.rs | 48 +-- .../component_events_dropped.rs | 8 +- .../component_events_timed_out.rs | 10 +- .../src/internal_event/events_received.rs | 12 +- .../src/internal_event/events_sent.rs | 18 +- .../src/internal_event/metric_name.rs | 337 ++++++++++++++++++ lib/vector-common/src/internal_event/mod.rs | 2 + .../src/internal_event/service.rs | 8 +- lib/vector-common/src/lib.rs | 57 +++ lib/vector-core/Cargo.toml | 1 + lib/vector-core/src/latency.rs | 12 +- lib/vector-core/src/metrics/mod.rs | 76 ++-- lib/vector-core/src/source_sender/builder.rs | 2 +- lib/vector-core/src/source_sender/mod.rs | 8 +- lib/vector-core/src/source_sender/sender.rs | 4 +- lib/vector-lib/src/lib.rs | 6 +- scripts/check-events | 11 + .../memory/internal_events.rs | 32 +- src/internal_events/adaptive_concurrency.rs | 17 +- src/internal_events/aggregate.rs | 13 +- src/internal_events/amqp.rs | 15 +- src/internal_events/apache_metrics.rs | 11 +- src/internal_events/api.rs | 9 +- src/internal_events/aws.rs | 9 +- src/internal_events/aws_cloudwatch_logs.rs | 7 +- src/internal_events/aws_ec2_metadata.rs | 13 +- src/internal_events/aws_ecs_metrics.rs | 13 +- src/internal_events/aws_kinesis.rs | 8 +- src/internal_events/aws_kinesis_firehose.rs | 11 +- src/internal_events/aws_sqs.rs | 42 ++- src/internal_events/batch.rs | 7 +- src/internal_events/common.rs | 19 +- src/internal_events/conditions.rs | 9 +- src/internal_events/datadog_agent.rs | 9 +- src/internal_events/datadog_metrics.rs | 7 +- src/internal_events/datadog_traces.rs | 9 +- src/internal_events/dedupe.rs | 6 +- src/internal_events/demo_logs.rs | 3 +- src/internal_events/dnstap.rs | 9 +- src/internal_events/docker_logs.rs | 23 +- src/internal_events/doris.rs | 12 +- src/internal_events/encoding_transcode.rs | 13 +- src/internal_events/eventstoredb_metrics.rs | 11 +- src/internal_events/exec.rs | 22 +- src/internal_events/expansion.rs | 7 +- src/internal_events/file.rs | 72 ++-- src/internal_events/file_descriptor.rs | 9 +- src/internal_events/fluent.rs | 9 +- src/internal_events/gcp_pubsub.rs | 13 +- src/internal_events/grpc.rs | 17 +- src/internal_events/heartbeat.rs | 11 +- src/internal_events/host_metrics.rs | 13 +- src/internal_events/http.rs | 25 +- src/internal_events/http_client.rs | 22 +- src/internal_events/http_client_source.rs | 13 +- src/internal_events/influxdb.rs | 7 +- src/internal_events/internal_logs.rs | 14 +- src/internal_events/journald.rs | 14 +- src/internal_events/kafka.rs | 39 +- src/internal_events/kubernetes_logs.rs | 34 +- src/internal_events/log_to_metric.rs | 19 +- src/internal_events/logplex.rs | 9 +- src/internal_events/loki.rs | 13 +- src/internal_events/lua.rs | 11 +- src/internal_events/metric_to_log.rs | 7 +- src/internal_events/mongodb_metrics.rs | 13 +- src/internal_events/mqtt.rs | 9 +- src/internal_events/nginx_metrics.rs | 13 +- src/internal_events/open.rs | 11 +- src/internal_events/parser.rs | 15 +- src/internal_events/postgresql_metrics.rs | 9 +- src/internal_events/process.rs | 21 +- src/internal_events/prometheus.rs | 17 +- src/internal_events/pulsar.rs | 15 +- src/internal_events/redis.rs | 9 +- src/internal_events/reduce.rs | 11 +- src/internal_events/remap.rs | 8 +- src/internal_events/sample.rs | 6 +- src/internal_events/sematext_metrics.rs | 13 +- src/internal_events/socket.rs | 33 +- src/internal_events/splunk_hec.rs | 39 +- src/internal_events/statsd_sink.rs | 7 +- src/internal_events/tag_cardinality_limit.rs | 17 +- src/internal_events/tcp.rs | 19 +- src/internal_events/template.rs | 11 +- src/internal_events/throttle.rs | 9 +- src/internal_events/udp.rs | 17 +- src/internal_events/unix.rs | 15 +- src/internal_events/websocket.rs | 27 +- src/internal_events/websocket_server.rs | 23 +- src/internal_events/windows.rs | 19 +- src/internal_events/windows_event_log.rs | 11 +- src/internal_telemetry/allocations/mod.rs | 13 +- src/sources/internal_metrics.rs | 47 ++- src/topology/builder.rs | 8 +- src/utilization.rs | 7 +- 106 files changed, 1254 insertions(+), 711 deletions(-) create mode 100644 lib/vector-common/src/internal_event/metric_name.rs diff --git a/Cargo.lock b/Cargo.lock index 9bab182e05373..2e217d2d24c4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12939,6 +12939,7 @@ dependencies = [ "sqlx", "stream-cancel", "strip-ansi-escapes", + "strum 0.28.0", "sysinfo", "syslog", "tempfile", @@ -13064,6 +13065,7 @@ dependencies = [ "serde_json", "smallvec", "stream-cancel", + "strum 0.28.0", "tokio", "tracing 0.1.44", "vector-common-macros", @@ -13191,6 +13193,7 @@ dependencies = [ "smallvec", "snafu 0.9.0", "socket2 0.5.10", + "strum 0.28.0", "tokio", "tokio-openssl", "tokio-stream", diff --git a/Cargo.toml b/Cargo.toml index 904f540cd26a5..e9cf7da0f400b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -479,6 +479,7 @@ openssl-src = { version = "300", default-features = false, features = ["force-en [dev-dependencies] approx = "0.5.1" +strum.workspace = true assert_cmd = { version = "2.0.17", default-features = false } aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } base64 = "0.22.1" diff --git a/benches/metrics_snapshot.rs b/benches/metrics_snapshot.rs index 279911e5e65a4..4df220256f7b9 100644 --- a/benches/metrics_snapshot.rs +++ b/benches/metrics_snapshot.rs @@ -1,4 +1,7 @@ use criterion::{BenchmarkId, Criterion, criterion_group}; +use strum::IntoEnumIterator; +use vector_lib::counter; +use vector_lib::internal_event::CounterName; fn benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("metrics_snapshot"); @@ -22,8 +25,9 @@ fn prepare_metrics(cardinality: usize) -> &'static vector::metrics::Controller { let controller = vector::metrics::Controller::get().unwrap(); controller.reset(); + let name = CounterName::iter().next().unwrap(); for idx in 0..cardinality { - metrics::counter!("test", "idx" => idx.to_string()).increment(1); + counter!(name, "idx" => idx.to_string()).increment(1); } controller diff --git a/clippy.toml b/clippy.toml index cbd8c640fdb71..4aaa1eb15b360 100644 --- a/clippy.toml +++ b/clippy.toml @@ -8,6 +8,12 @@ disallowed-methods = [ { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." }, ] +disallowed-macros = [ + { path = "metrics::counter", reason = "Use the `counter!` macro from `vector_common` with a `CounterName` variant instead of a raw string." }, + { path = "metrics::histogram", reason = "Use the `histogram!` macro from `vector_common` with a `HistogramName` variant instead of a raw string." }, + { path = "metrics::gauge", reason = "Use the `gauge!` macro from `vector_common` with a `GaugeName` variant instead of a raw string." }, +] + disallowed-types = [ { path = "once_cell::sync::OnceCell", reason = "Use `std::sync::OnceLock` instead." }, { path = "once_cell::unsync::OnceCell", reason = "Use `std::cell::OnceCell` instead." }, diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index d21119419887d..0518247e523a3 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -1,9 +1,12 @@ //! Internal events for codecs. -use metrics::counter; use tracing::error; -use vector_common::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type, +use vector_common::{ + counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, emit, error_stage, + error_type, + }, }; use vector_common_macros::NamedInternalEvent; @@ -24,7 +27,7 @@ impl InternalEvent for DecoderFramingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "decoder_frame", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -50,7 +53,7 @@ impl InternalEvent for DecoderDeserializeError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "decoder_deserialize", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -77,7 +80,7 @@ impl InternalEvent for EncoderFramingError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoder_frame", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -105,7 +108,7 @@ impl InternalEvent for EncoderSerializeError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoder_serialize", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -137,7 +140,7 @@ impl InternalEvent for EncoderWriteError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, ) @@ -170,7 +173,7 @@ impl InternalEvent for EncoderNullConstraintError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoding_null_constraint", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -201,7 +204,7 @@ impl InternalEvent for EncoderRecordBatchError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -228,7 +231,7 @@ impl InternalEvent for SchemaGenerationError<'_> { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parquet_schema_generation_failed", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -255,7 +258,7 @@ impl InternalEvent for ArrowWriterError<'_> { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parquet_arrow_writer_failed", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -281,7 +284,7 @@ impl InternalEvent for JsonSerializationError<'_> { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index 6d2735c7bb75f..31a6089322d0c 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -1,9 +1,10 @@ use std::time::Duration; -use metrics::{Histogram, counter, gauge, histogram}; +use metrics::Histogram; use vector_common::NamedInternalEvent; use vector_common::{ - internal_event::{InternalEvent, error_type}, + counter, gauge, histogram, + internal_event::{CounterName, GaugeName, HistogramName, InternalEvent, error_type}, registered_event, }; @@ -21,14 +22,14 @@ impl InternalEvent for BufferCreated { let stage = self.idx.to_string(); if self.max_size_events != 0 { gauge!( - "buffer_max_size_events", + GaugeName::BufferMaxSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) .set(self.max_size_events as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_max_event_size", + GaugeName::BufferMaxEventSize, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) @@ -36,14 +37,14 @@ impl InternalEvent for BufferCreated { } if self.max_size_bytes != 0 { gauge!( - "buffer_max_size_bytes", + GaugeName::BufferMaxSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) .set(self.max_size_bytes as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_max_byte_size", + GaugeName::BufferMaxByteSize, "buffer_id" => self.buffer_id, "stage" => stage, ) @@ -66,40 +67,40 @@ impl InternalEvent for BufferEventsReceived { #[expect(clippy::cast_precision_loss)] fn emit(self) { counter!( - "buffer_received_events_total", + CounterName::BufferReceivedEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.count); counter!( - "buffer_received_bytes_total", + CounterName::BufferReceivedBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -121,39 +122,39 @@ impl InternalEvent for BufferEventsSent { #[expect(clippy::cast_precision_loss)] fn emit(self) { counter!( - "buffer_sent_events_total", + CounterName::BufferSentEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.count); counter!( - "buffer_sent_bytes_total", + CounterName::BufferSentBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -200,14 +201,14 @@ impl InternalEvent for BufferEventsDropped { } counter!( - "buffer_discarded_events_total", + CounterName::BufferDiscardedEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string(), "intentional" => intentional_str, ) .increment(self.count); counter!( - "buffer_discarded_bytes_total", + CounterName::BufferDiscardedBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string(), "intentional" => intentional_str, @@ -215,26 +216,26 @@ impl InternalEvent for BufferEventsDropped { .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -258,7 +259,7 @@ impl InternalEvent for BufferReadError { stage = "processing", ); counter!( - "buffer_errors_total", "error_code" => self.error_code, + CounterName::BufferErrorsTotal, "error_code" => self.error_code, "error_type" => "reader_failed", "stage" => "processing", ) @@ -270,7 +271,7 @@ registered_event! { BufferSendDuration { stage: usize, } => { - send_duration: Histogram = histogram!("buffer_send_duration_seconds", "stage" => self.stage.to_string()), + send_duration: Histogram = histogram!(HistogramName::BufferSendDurationSeconds, "stage" => self.stage.to_string()), } fn emit(&self, duration: Duration) { diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index a246aa9659812..3e1f3f7c98c93 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -127,6 +127,7 @@ struct Metrics { impl Metrics { #[expect(clippy::cast_precision_loss)] // We have to convert buffer sizes for a gauge, it's okay to lose precision here. + #[allow(clippy::disallowed_macros)] // Metric names are constructed dynamically from runtime prefixes. fn new( limit: MemoryBufferSize, metadata: ChannelMetricMetadata, diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 95098f19b1bd3..a583641c68cec 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,6 +48,7 @@ pin-project.workspace = true serde.workspace = true serde_json.workspace = true smallvec = { version = "1", default-features = false } +strum.workspace = true stream-cancel = { version = "0.8.2", default-features = false } tokio = { workspace = true, features = ["macros", "time"] } tracing.workspace = true diff --git a/lib/vector-common/src/internal_event/bytes_received.rs b/lib/vector-common/src/internal_event/bytes_received.rs index 98044ecb84570..4264c3cbc365a 100644 --- a/lib/vector-common/src/internal_event/bytes_received.rs +++ b/lib/vector-common/src/internal_event/bytes_received.rs @@ -1,12 +1,14 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::{ByteSize, Protocol, SharedString}; +use crate::counter; + +use super::{ByteSize, CounterName, Protocol, SharedString}; crate::registered_event!( BytesReceived { protocol: SharedString, } => { - received_bytes: Counter = counter!("component_received_bytes_total", "protocol" => self.protocol.clone()), + received_bytes: Counter = counter!(CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol.clone()), protocol: SharedString = self.protocol, } diff --git a/lib/vector-common/src/internal_event/bytes_sent.rs b/lib/vector-common/src/internal_event/bytes_sent.rs index 0b2a88247f96c..b363893783f43 100644 --- a/lib/vector-common/src/internal_event/bytes_sent.rs +++ b/lib/vector-common/src/internal_event/bytes_sent.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, counter}; +use metrics::Counter; + +use crate::counter; use tracing::trace; -use super::{ByteSize, Protocol, SharedString}; +use super::{ByteSize, CounterName, Protocol, SharedString}; crate::registered_event!( BytesSent { protocol: SharedString, } => { - bytes_sent: Counter = counter!("component_sent_bytes_total", "protocol" => self.protocol.clone()), + bytes_sent: Counter = counter!(CounterName::ComponentSentBytesTotal, "protocol" => self.protocol.clone()), protocol: SharedString = self.protocol, } diff --git a/lib/vector-common/src/internal_event/cached_event.rs b/lib/vector-common/src/internal_event/cached_event.rs index 782faeb5a1d01..ec3d492998ab4 100644 --- a/lib/vector-common/src/internal_event/cached_event.rs +++ b/lib/vector-common/src/internal_event/cached_event.rs @@ -91,34 +91,36 @@ where #[cfg(test)] mod tests { #![allow(unreachable_pub)] - use metrics::{Counter, counter}; + use metrics::Counter; + use strum::IntoEnumIterator; use super::*; - - crate::registered_event!( - TestEvent { - fixed: String, - dynamic: String, - } => { - event: Counter = { - counter!("test_event_total", "fixed" => self.fixed, "dynamic" => self.dynamic) - }, - } - - fn emit(&self, count: u64) { - self.event.increment(count); - } - - fn register(fixed: String, dynamic: String) { - crate::internal_event::register(TestEvent { - fixed, - dynamic, - }) - } - ); + use crate::internal_event::CounterName; #[test] fn test_fixed_tag() { + crate::registered_event!( + TestEvent { + fixed: String, + dynamic: String, + } => { + event: Counter = { + crate::counter!(CounterName::iter().next().unwrap(), "fixed" => self.fixed, "dynamic" => self.dynamic) + }, + } + + fn emit(&self, count: u64) { + self.event.increment(count); + } + + fn register(fixed: String, dynamic: String) { + crate::internal_event::register(TestEvent { + fixed, + dynamic, + }) + } + ); + let event: RegisteredEventCache = RegisteredEventCache::new("fixed".to_string()); diff --git a/lib/vector-common/src/internal_event/component_events_dropped.rs b/lib/vector-common/src/internal_event/component_events_dropped.rs index bf45c054e45e1..8421d1d0f49ae 100644 --- a/lib/vector-common/src/internal_event/component_events_dropped.rs +++ b/lib/vector-common/src/internal_event/component_events_dropped.rs @@ -1,6 +1,8 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::{Count, InternalEvent, InternalEventHandle, RegisterInternalEvent}; +use crate::counter; + +use super::{Count, CounterName, InternalEvent, InternalEventHandle, RegisterInternalEvent}; use crate::NamedInternalEvent; pub const INTENTIONAL: bool = true; @@ -32,7 +34,7 @@ impl<'a, const INTENTIONAL: bool> RegisterInternalEvent fn register(self) -> Self::Handle { Self::Handle { discarded_events: counter!( - "component_discarded_events_total", + CounterName::ComponentDiscardedEventsTotal, "intentional" => if INTENTIONAL { "true" } else { "false" }, ), reason: self.reason, diff --git a/lib/vector-common/src/internal_event/component_events_timed_out.rs b/lib/vector-common/src/internal_event/component_events_timed_out.rs index bf138dd1481c7..4662322fe9776 100644 --- a/lib/vector-common/src/internal_event/component_events_timed_out.rs +++ b/lib/vector-common/src/internal_event/component_events_timed_out.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::Count; +use crate::counter; + +use super::{Count, CounterName}; crate::registered_event! { ComponentEventsTimedOut { reason: &'static str, } => { - timed_out_events: Counter = counter!("component_timed_out_events_total"), - timed_out_requests: Counter = counter!("component_timed_out_requests_total"), + timed_out_events: Counter = counter!(CounterName::ComponentTimedOutEventsTotal), + timed_out_requests: Counter = counter!(CounterName::ComponentTimedOutRequestsTotal), reason: &'static str = self.reason, } diff --git a/lib/vector-common/src/internal_event/events_received.rs b/lib/vector-common/src/internal_event/events_received.rs index 40ba999a8cfca..a08ebc6e8438b 100644 --- a/lib/vector-common/src/internal_event/events_received.rs +++ b/lib/vector-common/src/internal_event/events_received.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, Histogram, counter, histogram}; +use metrics::{Counter, Histogram}; + +use crate::{counter, histogram}; use tracing::trace; -use super::CountByteSize; +use super::{CountByteSize, CounterName, HistogramName}; crate::registered_event!( EventsReceived => { - events_count: Histogram = histogram!("component_received_events_count"), - events: Counter = counter!("component_received_events_total"), - event_bytes: Counter = counter!("component_received_event_bytes_total"), + events_count: Histogram = histogram!(HistogramName::ComponentReceivedEventsCount), + events: Counter = counter!(CounterName::ComponentReceivedEventsTotal), + event_bytes: Counter = counter!(CounterName::ComponentReceivedEventBytesTotal), } fn emit(&self, data: CountByteSize) { diff --git a/lib/vector-common/src/internal_event/events_sent.rs b/lib/vector-common/src/internal_event/events_sent.rs index 21e11260b2d38..3ae63db761381 100644 --- a/lib/vector-common/src/internal_event/events_sent.rs +++ b/lib/vector-common/src/internal_event/events_sent.rs @@ -1,9 +1,11 @@ use std::sync::Arc; -use metrics::{Counter, counter}; +use metrics::Counter; + +use crate::counter; use tracing::trace; -use super::{CountByteSize, OptionalTag, Output, SharedString}; +use super::{CountByteSize, CounterName, OptionalTag, Output, SharedString}; use crate::config::ComponentKey; pub const DEFAULT_OUTPUT: &str = "_default"; @@ -13,14 +15,14 @@ crate::registered_event!( output: Option, } => { events: Counter = if let Some(output) = &self.output { - counter!("component_sent_events_total", "output" => output.clone()) + counter!(CounterName::ComponentSentEventsTotal, "output" => output.clone()) } else { - counter!("component_sent_events_total") + counter!(CounterName::ComponentSentEventsTotal) }, event_bytes: Counter = if let Some(output) = &self.output { - counter!("component_sent_event_bytes_total", "output" => output.clone()) + counter!(CounterName::ComponentSentEventBytesTotal, "output" => output.clone()) } else { - counter!("component_sent_event_bytes_total") + counter!(CounterName::ComponentSentEventBytesTotal) }, output: Option = self.output, } @@ -75,10 +77,10 @@ crate::registered_event!( service: OptionalTag, } => { events: Counter = { - counter!("component_sent_events_total", &make_tags(&self.source, &self.service)) + counter!(CounterName::ComponentSentEventsTotal, &make_tags(&self.source, &self.service)) }, event_bytes: Counter = { - counter!("component_sent_event_bytes_total", &make_tags(&self.source, &self.service)) + counter!(CounterName::ComponentSentEventBytesTotal, &make_tags(&self.source, &self.service)) }, } diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs new file mode 100644 index 0000000000000..21314695a13bf --- /dev/null +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -0,0 +1,337 @@ +use strum::{AsRefStr, Display, EnumIter}; + +/// Canonical list of all per-component internal metric names emitted by Vector. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum CounterName { + ComponentReceivedEventsTotal, + ComponentReceivedEventBytesTotal, + ComponentReceivedBytesTotal, + ComponentSentEventsTotal, + ComponentSentEventBytesTotal, + ComponentSentBytesTotal, + ComponentDiscardedEventsTotal, + ComponentErrorsTotal, + ComponentTimedOutEventsTotal, + ComponentTimedOutRequestsTotal, + BufferReceivedEventsTotal, + BufferReceivedBytesTotal, + BufferSentEventsTotal, + BufferSentBytesTotal, + BufferDiscardedEventsTotal, + BufferDiscardedBytesTotal, + BufferErrorsTotal, + // Internal events from src/internal_events/ + AggregateEventsRecordedTotal, + AggregateFailedUpdates, + AggregateFlushesTotal, + ApiStartedTotal, + CheckpointsTotal, + ChecksumErrorsTotal, + CollectCompletedTotal, + CommandExecutedTotal, + ConnectionEstablishedTotal, + ConnectionSendErrorsTotal, + ConnectionShutdownTotal, + ContainerProcessedEventsTotal, + ContainersUnwatchedTotal, + ContainersWatchedTotal, + DecoderBomRemovalsTotal, + DecoderMalformedReplacementWarningsTotal, + DorisBytesLoadedTotal, + DorisRowsFilteredTotal, + DorisRowsLoadedTotal, + EncoderUnmappableReplacementWarningsTotal, + EventsDiscardedTotal, + FilesAddedTotal, + FilesDeletedTotal, + FilesResumedTotal, + FilesUnwatchedTotal, + GrpcServerMessagesReceivedTotal, + GrpcServerMessagesSentTotal, + HttpClientErrorsTotal, + HttpClientRequestsSentTotal, + HttpClientResponsesTotal, + HttpServerRequestsReceivedTotal, + HttpServerResponsesSentTotal, + KafkaConsumedMessagesBytesTotal, + KafkaConsumedMessagesTotal, + KafkaProducedMessagesBytesTotal, + KafkaProducedMessagesTotal, + KafkaRequestsBytesTotal, + KafkaRequestsTotal, + KafkaResponsesBytesTotal, + KafkaResponsesTotal, + MetadataRefreshFailedTotal, + MetadataRefreshSuccessfulTotal, + ParseErrorsTotal, + QuitTotal, + ReloadedTotal, + RewrittenTimestampEventsTotal, + SqsMessageDeferSucceededTotal, + SqsMessageDeleteSucceededTotal, + SqsMessageProcessingSucceededTotal, + SqsMessageReceiveSucceededTotal, + SqsMessageReceivedMessagesTotal, + StaleEventsFlushedTotal, + StartedTotal, + StoppedTotal, + TagValueLimitExceededTotal, + ValueLimitReachedTotal, + WebsocketBytesSentTotal, + WebsocketMessagesSentTotal, + WindowsServiceInstallTotal, + WindowsServiceRestartTotal, + WindowsServiceStartTotal, + WindowsServiceStopTotal, + WindowsServiceUninstallTotal, + K8sEventNamespaceAnnotationFailuresTotal, + K8sEventNodeAnnotationFailuresTotal, + K8sFormatPickerEdgeCasesTotal, + K8sDockerFormatParseFailuresTotal, + SqsS3EventRecordIgnoredTotal, + ComponentAllocatedBytesTotal, + ComponentDeallocatedBytesTotal, + MemoryEnrichmentTableFailedInsertions, + MemoryEnrichmentTableFailedReads, + MemoryEnrichmentTableFlushesTotal, + MemoryEnrichmentTableInsertionsTotal, + MemoryEnrichmentTableReadsTotal, + MemoryEnrichmentTableTtlExpirations, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum HistogramName { + ComponentReceivedEventsCount, + ComponentReceivedBytes, + BufferSendDurationSeconds, + ComponentLatencySeconds, + SourceLagTimeSeconds, + SourceSendLatencySeconds, + SourceSendBatchLatencySeconds, + AdaptiveConcurrencyAveragedRtt, + AdaptiveConcurrencyBackPressure, + AdaptiveConcurrencyInFlight, + AdaptiveConcurrencyLimit, + AdaptiveConcurrencyObservedRtt, + AdaptiveConcurrencyPastRttMean, + AdaptiveConcurrencyReachedLimit, + S3ObjectProcessingSucceededDurationSeconds, + S3ObjectProcessingFailedDurationSeconds, + CollectDurationSeconds, + CommandExecutionDurationSeconds, + GrpcServerHandlerDurationSeconds, + HttpServerHandlerDurationSeconds, + HttpClientRttSeconds, + HttpClientResponseRttSeconds, + HttpClientErrorRttSeconds, +} + +impl HistogramName { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentReceivedEventsCount => "component_received_events_count", + Self::ComponentReceivedBytes => "component_received_bytes", + Self::BufferSendDurationSeconds => "buffer_send_duration_seconds", + Self::ComponentLatencySeconds => "component_latency_seconds", + Self::SourceLagTimeSeconds => "source_lag_time_seconds", + Self::SourceSendLatencySeconds => "source_send_latency_seconds", + Self::SourceSendBatchLatencySeconds => "source_send_batch_latency_seconds", + Self::AdaptiveConcurrencyAveragedRtt => "adaptive_concurrency_averaged_rtt", + Self::AdaptiveConcurrencyBackPressure => "adaptive_concurrency_back_pressure", + Self::AdaptiveConcurrencyInFlight => "adaptive_concurrency_in_flight", + Self::AdaptiveConcurrencyLimit => "adaptive_concurrency_limit", + Self::AdaptiveConcurrencyObservedRtt => "adaptive_concurrency_observed_rtt", + Self::AdaptiveConcurrencyPastRttMean => "adaptive_concurrency_past_rtt_mean", + Self::AdaptiveConcurrencyReachedLimit => "adaptive_concurrency_reached_limit", + Self::S3ObjectProcessingSucceededDurationSeconds => { + "s3_object_processing_succeeded_duration_seconds" + } + Self::S3ObjectProcessingFailedDurationSeconds => { + "s3_object_processing_failed_duration_seconds" + } + Self::CollectDurationSeconds => "collect_duration_seconds", + Self::CommandExecutionDurationSeconds => "command_execution_duration_seconds", + Self::GrpcServerHandlerDurationSeconds => "grpc_server_handler_duration_seconds", + Self::HttpServerHandlerDurationSeconds => "http_server_handler_duration_seconds", + Self::HttpClientRttSeconds => "http_client_rtt_seconds", + Self::HttpClientResponseRttSeconds => "http_client_response_rtt_seconds", + Self::HttpClientErrorRttSeconds => "http_client_error_rtt_seconds", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum GaugeName { + ComponentLatencyMeanSeconds, + BufferMaxSizeEvents, + BufferMaxEventSize, + BufferMaxSizeBytes, + BufferMaxByteSize, + BufferEvents, + BufferSizeEvents, + BufferSizeBytes, + BufferByteSize, + Utilization, + ComponentAllocatedBytes, + OpenFiles, + UptimeSeconds, + BuildInfo, + KafkaQueueMessages, + KafkaQueueMessagesBytes, + KafkaConsumerLag, + LuaMemoryUsedBytes, + OpenConnections, + ActiveEndpoints, + SplunkPendingAcks, + ActiveClients, + MemoryEnrichmentTableObjectsCount, + MemoryEnrichmentTableByteSize, +} + +impl GaugeName { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentLatencyMeanSeconds => "component_latency_mean_seconds", + Self::BufferMaxSizeEvents => "buffer_max_size_events", + Self::BufferMaxEventSize => "buffer_max_event_size", + Self::BufferMaxSizeBytes => "buffer_max_size_bytes", + Self::BufferMaxByteSize => "buffer_max_byte_size", + Self::BufferEvents => "buffer_events", + Self::BufferSizeEvents => "buffer_size_events", + Self::BufferSizeBytes => "buffer_size_bytes", + Self::BufferByteSize => "buffer_byte_size", + Self::Utilization => "utilization", + Self::ComponentAllocatedBytes => "component_allocated_bytes", + Self::OpenFiles => "open_files", + Self::UptimeSeconds => "uptime_seconds", + Self::BuildInfo => "build_info", + Self::KafkaQueueMessages => "kafka_queue_messages", + Self::KafkaQueueMessagesBytes => "kafka_queue_messages_bytes", + Self::KafkaConsumerLag => "kafka_consumer_lag", + Self::LuaMemoryUsedBytes => "lua_memory_used_bytes", + Self::OpenConnections => "open_connections", + Self::ActiveEndpoints => "active_endpoints", + Self::SplunkPendingAcks => "splunk_pending_acks", + Self::ActiveClients => "active_clients", + Self::MemoryEnrichmentTableObjectsCount => "memory_enrichment_table_objects_count", + Self::MemoryEnrichmentTableByteSize => "memory_enrichment_table_byte_size", + } + } +} + +impl CounterName { + #[allow(clippy::too_many_lines)] + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentReceivedEventsTotal => "component_received_events_total", + Self::ComponentReceivedEventBytesTotal => "component_received_event_bytes_total", + Self::ComponentReceivedBytesTotal => "component_received_bytes_total", + Self::ComponentSentEventsTotal => "component_sent_events_total", + Self::ComponentSentEventBytesTotal => "component_sent_event_bytes_total", + Self::ComponentSentBytesTotal => "component_sent_bytes_total", + Self::ComponentDiscardedEventsTotal => "component_discarded_events_total", + Self::ComponentErrorsTotal => "component_errors_total", + Self::ComponentTimedOutEventsTotal => "component_timed_out_events_total", + Self::ComponentTimedOutRequestsTotal => "component_timed_out_requests_total", + Self::BufferReceivedEventsTotal => "buffer_received_events_total", + Self::BufferReceivedBytesTotal => "buffer_received_bytes_total", + Self::BufferSentEventsTotal => "buffer_sent_events_total", + Self::BufferSentBytesTotal => "buffer_sent_bytes_total", + Self::BufferDiscardedEventsTotal => "buffer_discarded_events_total", + Self::BufferDiscardedBytesTotal => "buffer_discarded_bytes_total", + Self::BufferErrorsTotal => "buffer_errors_total", + Self::AggregateEventsRecordedTotal => "aggregate_events_recorded_total", + Self::AggregateFailedUpdates => "aggregate_failed_updates", + Self::AggregateFlushesTotal => "aggregate_flushes_total", + Self::ApiStartedTotal => "api_started_total", + Self::CheckpointsTotal => "checkpoints_total", + Self::ChecksumErrorsTotal => "checksum_errors_total", + Self::CollectCompletedTotal => "collect_completed_total", + Self::CommandExecutedTotal => "command_executed_total", + Self::ConnectionEstablishedTotal => "connection_established_total", + Self::ConnectionSendErrorsTotal => "connection_send_errors_total", + Self::ConnectionShutdownTotal => "connection_shutdown_total", + Self::ContainerProcessedEventsTotal => "container_processed_events_total", + Self::ContainersUnwatchedTotal => "containers_unwatched_total", + Self::ContainersWatchedTotal => "containers_watched_total", + Self::DecoderBomRemovalsTotal => "decoder_bom_removals_total", + Self::DecoderMalformedReplacementWarningsTotal => { + "decoder_malformed_replacement_warnings_total" + } + Self::DorisBytesLoadedTotal => "doris_bytes_loaded_total", + Self::DorisRowsFilteredTotal => "doris_rows_filtered_total", + Self::DorisRowsLoadedTotal => "doris_rows_loaded_total", + Self::EncoderUnmappableReplacementWarningsTotal => { + "encoder_unmappable_replacement_warnings_total" + } + Self::EventsDiscardedTotal => "events_discarded_total", + Self::FilesAddedTotal => "files_added_total", + Self::FilesDeletedTotal => "files_deleted_total", + Self::FilesResumedTotal => "files_resumed_total", + Self::FilesUnwatchedTotal => "files_unwatched_total", + Self::GrpcServerMessagesReceivedTotal => "grpc_server_messages_received_total", + Self::GrpcServerMessagesSentTotal => "grpc_server_messages_sent_total", + Self::HttpClientErrorsTotal => "http_client_errors_total", + Self::HttpClientRequestsSentTotal => "http_client_requests_sent_total", + Self::HttpClientResponsesTotal => "http_client_responses_total", + Self::HttpServerRequestsReceivedTotal => "http_server_requests_received_total", + Self::HttpServerResponsesSentTotal => "http_server_responses_sent_total", + Self::KafkaConsumedMessagesBytesTotal => "kafka_consumed_messages_bytes_total", + Self::KafkaConsumedMessagesTotal => "kafka_consumed_messages_total", + Self::KafkaProducedMessagesBytesTotal => "kafka_produced_messages_bytes_total", + Self::KafkaProducedMessagesTotal => "kafka_produced_messages_total", + Self::KafkaRequestsBytesTotal => "kafka_requests_bytes_total", + Self::KafkaRequestsTotal => "kafka_requests_total", + Self::KafkaResponsesBytesTotal => "kafka_responses_bytes_total", + Self::KafkaResponsesTotal => "kafka_responses_total", + Self::MetadataRefreshFailedTotal => "metadata_refresh_failed_total", + Self::MetadataRefreshSuccessfulTotal => "metadata_refresh_successful_total", + Self::ParseErrorsTotal => "parse_errors_total", + Self::QuitTotal => "quit_total", + Self::ReloadedTotal => "reloaded_total", + Self::RewrittenTimestampEventsTotal => "rewritten_timestamp_events_total", + Self::SqsMessageDeferSucceededTotal => "sqs_message_defer_succeeded_total", + Self::SqsMessageDeleteSucceededTotal => "sqs_message_delete_succeeded_total", + Self::SqsMessageProcessingSucceededTotal => "sqs_message_processing_succeeded_total", + Self::SqsMessageReceiveSucceededTotal => "sqs_message_receive_succeeded_total", + Self::SqsMessageReceivedMessagesTotal => "sqs_message_received_messages_total", + Self::StaleEventsFlushedTotal => "stale_events_flushed_total", + Self::StartedTotal => "started_total", + Self::StoppedTotal => "stopped_total", + Self::TagValueLimitExceededTotal => "tag_value_limit_exceeded_total", + Self::ValueLimitReachedTotal => "value_limit_reached_total", + Self::WebsocketBytesSentTotal => "websocket_bytes_sent_total", + Self::WebsocketMessagesSentTotal => "websocket_messages_sent_total", + Self::WindowsServiceInstallTotal => "windows_service_install_total", + Self::WindowsServiceRestartTotal => "windows_service_restart_total", + Self::WindowsServiceStartTotal => "windows_service_start_total", + Self::WindowsServiceStopTotal => "windows_service_stop_total", + Self::WindowsServiceUninstallTotal => "windows_service_uninstall_total", + Self::K8sEventNamespaceAnnotationFailuresTotal => { + "k8s_event_namespace_annotation_failures_total" + } + Self::K8sEventNodeAnnotationFailuresTotal => "k8s_event_node_annotation_failures_total", + Self::K8sFormatPickerEdgeCasesTotal => "k8s_format_picker_edge_cases_total", + Self::K8sDockerFormatParseFailuresTotal => "k8s_docker_format_parse_failures_total", + Self::SqsS3EventRecordIgnoredTotal => "sqs_s3_event_record_ignored_total", + Self::ComponentAllocatedBytesTotal => "component_allocated_bytes_total", + Self::ComponentDeallocatedBytesTotal => "component_deallocated_bytes_total", + Self::MemoryEnrichmentTableFailedInsertions => { + "memory_enrichment_table_failed_insertions" + } + Self::MemoryEnrichmentTableFailedReads => "memory_enrichment_table_failed_reads", + Self::MemoryEnrichmentTableFlushesTotal => "memory_enrichment_table_flushes_total", + Self::MemoryEnrichmentTableInsertionsTotal => { + "memory_enrichment_table_insertions_total" + } + Self::MemoryEnrichmentTableReadsTotal => "memory_enrichment_table_reads_total", + Self::MemoryEnrichmentTableTtlExpirations => "memory_enrichment_table_ttl_expirations", + } + } +} diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index 272e2344900e3..e776e89553172 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -5,6 +5,7 @@ pub mod component_events_dropped; pub mod component_events_timed_out; mod events_received; mod events_sent; +pub mod metric_name; mod optional_tag; mod prelude; pub mod service; @@ -19,6 +20,7 @@ pub use component_events_dropped::{ComponentEventsDropped, INTENTIONAL, UNINTENT pub use component_events_timed_out::ComponentEventsTimedOut; pub use events_received::{EventsReceived, EventsReceivedHandle}; pub use events_sent::{DEFAULT_OUTPUT, EventsSent, TaggedEventsSent}; +pub use metric_name::{CounterName, GaugeName, HistogramName}; pub use metrics::SharedString; pub use optional_tag::OptionalTag; pub use prelude::{error_stage, error_type}; diff --git a/lib/vector-common/src/internal_event/service.rs b/lib/vector-common/src/internal_event/service.rs index 93a3c9b745722..60d615b3d61a0 100644 --- a/lib/vector-common/src/internal_event/service.rs +++ b/lib/vector-common/src/internal_event/service.rs @@ -1,4 +1,6 @@ -use metrics::counter; +use super::CounterName; + +use crate::counter; use super::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type}; use crate::NamedInternalEvent; @@ -17,7 +19,7 @@ impl InternalEvent for PollReadyError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) @@ -43,7 +45,7 @@ impl InternalEvent for CallError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index 099a068578090..75849aac75884 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -66,6 +66,63 @@ pub mod trigger; #[macro_use] extern crate tracing; +/// Typed wrapper around `metrics::counter!` that only accepts [`internal_event::CounterName`]. +#[macro_export] +macro_rules! counter { + ($name:expr) => {{ + let _name: $crate::internal_event::CounterName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::counter!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::CounterName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::counter!(_name.as_str(), $($rest)*) + } + }}; +} + +/// Typed wrapper around `metrics::histogram!` that only accepts [`internal_event::HistogramName`]. +#[macro_export] +macro_rules! histogram { + ($name:expr) => {{ + let _name: $crate::internal_event::HistogramName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::histogram!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::HistogramName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::histogram!(_name.as_str(), $($rest)*) + } + }}; +} + +/// Typed wrapper around `metrics::gauge!` that only accepts [`internal_event::GaugeName`]. +#[macro_export] +macro_rules! gauge { + ($name:expr) => {{ + let _name: $crate::internal_event::GaugeName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::gauge!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::GaugeName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::gauge!(_name.as_str(), $($rest)*) + } + }}; +} + /// Vector's basic error type, dynamically dispatched and safe to send across /// threads. pub type Error = Box; diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 5c3370ac1a526..edf7b43a4a74f 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -78,6 +78,7 @@ prost-build.workspace = true [dev-dependencies] base64 = "0.22.1" +strum.workspace = true chrono-tz.workspace = true criterion = { workspace = true, features = ["html_reports"] } env-test-util = "1.0.1" diff --git a/lib/vector-core/src/latency.rs b/lib/vector-core/src/latency.rs index 536a8ef532d0d..b25ca73427749 100644 --- a/lib/vector-core/src/latency.rs +++ b/lib/vector-core/src/latency.rs @@ -1,12 +1,14 @@ use std::time::Instant; -use metrics::{Histogram, gauge, histogram}; +use metrics::Histogram; use vector_common::stats::EwmaGauge; +use vector_common::{ + gauge, histogram, + internal_event::{GaugeName, HistogramName}, +}; use crate::event::EventArray; -const COMPONENT_LATENCY: &str = "component_latency_seconds"; -const COMPONENT_LATENCY_MEAN: &str = "component_latency_mean_seconds"; const DEFAULT_LATENCY_EWMA_ALPHA: f64 = 0.9; #[derive(Debug)] @@ -18,9 +20,9 @@ pub struct LatencyRecorder { impl LatencyRecorder { pub fn new(ewma_alpha: Option) -> Self { Self { - histogram: histogram!(COMPONENT_LATENCY), + histogram: histogram!(HistogramName::ComponentLatencySeconds), gauge: EwmaGauge::new( - gauge!(COMPONENT_LATENCY_MEAN), + gauge!(GaugeName::ComponentLatencyMeanSeconds), ewma_alpha.or(Some(DEFAULT_LATENCY_EWMA_ALPHA)), ), } diff --git a/lib/vector-core/src/metrics/mod.rs b/lib/vector-core/src/metrics/mod.rs index cd900d392f004..ea4c040163c7f 100644 --- a/lib/vector-core/src/metrics/mod.rs +++ b/lib/vector-core/src/metrics/mod.rs @@ -202,6 +202,12 @@ impl Controller { #[cfg(test)] mod tests { + use strum::IntoEnumIterator; + use vector_common::{ + counter, gauge, + internal_event::{CounterName, GaugeName}, + }; + use super::*; use crate::{ config::metrics_expiration::{ @@ -224,8 +230,9 @@ mod tests { let controller = Controller::get().unwrap(); controller.reset(); + let name = CounterName::iter().next().unwrap(); for idx in 0..cardinality { - metrics::counter!("test", "idx" => idx.to_string()).increment(1); + counter!(name, "idx" => idx.to_string()).increment(1); } let metrics = controller.capture_metrics(); @@ -253,11 +260,11 @@ mod tests { fn handles_registered_metrics() { let controller = init_metrics(); - let counter = metrics::counter!("test7"); + let counter = counter!(CounterName::iter().next().unwrap()); assert_eq!(controller.capture_metrics().len(), 3); counter.increment(1); assert_eq!(controller.capture_metrics().len(), 3); - let gauge = metrics::gauge!("test8"); + let gauge = gauge!(GaugeName::iter().next().unwrap()); assert_eq!(controller.capture_metrics().len(), 4); gauge.set(1.0); assert_eq!(controller.capture_metrics().len(), 4); @@ -270,12 +277,16 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + + counter!(name_a).increment(1); + counter!(name_b).increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test2").increment(3); + counter!(name_a).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } @@ -286,12 +297,13 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - metrics::counter!("test4", "tag" => "value1").increment(1); - metrics::counter!("test4", "tag" => "value2").increment(2); + let name = CounterName::iter().next().unwrap(); + counter!(name, "tag" => "value1").increment(1); + counter!(name, "tag" => "value2").increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test4", "tag" => "value1").increment(3); + counter!(name, "tag" => "value1").increment(3); assert_eq!(controller.capture_metrics().len(), 3); } @@ -302,8 +314,12 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - let a = metrics::counter!("test5"); - metrics::counter!("test6").increment(5); + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + + let a = counter!(name_a); + counter!(name_b).increment(5); assert_eq!(controller.capture_metrics().len(), 4); a.increment(1); assert_eq!(controller.capture_metrics().len(), 4); @@ -316,7 +332,7 @@ mod tests { assert_eq!(metrics.len(), 3); let metric = metrics .into_iter() - .find(|metric| metric.name() == "test5") + .find(|metric| metric.name() == name_a.as_str()) .expect("Test metric is not present"); match metric.value() { MetricValue::Counter { value } => assert_eq!(*value, 2.0), @@ -327,12 +343,17 @@ mod tests { #[test] fn expires_metrics_per_set() { let controller = init_metrics(); + + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + controller .set_expiry( None, vec![PerMetricSetExpiration { name: Some(MetricNameMatcherConfig::Exact { - value: "test3".to_string(), + value: name_b.as_str().to_string(), }), labels: None, expire_secs: IDLE_TIMEOUT, @@ -340,25 +361,32 @@ mod tests { ) .unwrap(); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); + counter!(name_a).increment(1); + counter!(name_b).increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test2").increment(3); + counter!(name_a).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } #[test] fn expires_metrics_multiple_different_sets() { let controller = init_metrics(); + + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + let name_c = names.next().unwrap(); + let name_d = names.next().unwrap(); + controller .set_expiry( Some(IDLE_TIMEOUT * 3.0), vec![ PerMetricSetExpiration { name: Some(MetricNameMatcherConfig::Exact { - value: "test3".to_string(), + value: name_c.as_str().to_string(), }), labels: None, expire_secs: IDLE_TIMEOUT, @@ -377,22 +405,22 @@ mod tests { ) .unwrap(); - metrics::counter!("test1").increment(1); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); - metrics::counter!("test4", "tag" => "value1").increment(3); + counter!(name_a).increment(1); + counter!(name_b).increment(1); + counter!(name_c).increment(2); + counter!(name_d, "tag" => "value1").increment(3); assert_eq!(controller.capture_metrics().len(), 6); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 1.5)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 5); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } } diff --git a/lib/vector-core/src/source_sender/builder.rs b/lib/vector-core/src/source_sender/builder.rs index f3051726a7f56..7f9f5d522497a 100644 --- a/lib/vector-core/src/source_sender/builder.rs +++ b/lib/vector-core/src/source_sender/builder.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, time::Duration}; -use metrics::histogram; use vector_buffers::topology::channel::LimitedReceiver; +use vector_common::histogram; use vector_common::internal_event::DEFAULT_OUTPUT; use super::{ diff --git a/lib/vector-core/src/source_sender/mod.rs b/lib/vector-core/src/source_sender/mod.rs index fc0cb417890a0..24eb23688c0a1 100644 --- a/lib/vector-core/src/source_sender/mod.rs +++ b/lib/vector-core/src/source_sender/mod.rs @@ -22,6 +22,8 @@ pub const CHUNK_SIZE: usize = 1000; #[cfg(any(test, feature = "test"))] const TEST_BUFFER_SIZE: usize = 100; -const LAG_TIME_NAME: &str = "source_lag_time_seconds"; -const SEND_LATENCY_NAME: &str = "source_send_latency_seconds"; -const SEND_BATCH_LATENCY_NAME: &str = "source_send_batch_latency_seconds"; +use vector_common::internal_event::HistogramName; + +const LAG_TIME_NAME: HistogramName = HistogramName::SourceLagTimeSeconds; +const SEND_LATENCY_NAME: HistogramName = HistogramName::SourceSendLatencySeconds; +const SEND_BATCH_LATENCY_NAME: HistogramName = HistogramName::SourceSendBatchLatencySeconds; diff --git a/lib/vector-core/src/source_sender/sender.rs b/lib/vector-core/src/source_sender/sender.rs index 6041dad3dad9f..d5834ab0edff5 100644 --- a/lib/vector-core/src/source_sender/sender.rs +++ b/lib/vector-core/src/source_sender/sender.rs @@ -5,12 +5,12 @@ use std::{collections::HashMap, time::Instant}; use futures::Stream; #[cfg(any(test, feature = "test"))] use futures::StreamExt as _; -#[cfg(any(test, feature = "test"))] -use metrics::histogram; use vector_buffers::EventCount; #[cfg(any(test, feature = "test"))] use vector_buffers::topology::channel::LimitedReceiver; #[cfg(any(test, feature = "test"))] +use vector_common::histogram; +#[cfg(any(test, feature = "test"))] use vector_common::internal_event::DEFAULT_OUTPUT; #[cfg(doc)] use vector_common::internal_event::{ComponentEventsDropped, EventsSent}; diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index 82bb67cf9afce..277d28df18e92 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -11,9 +11,9 @@ pub use vector_buffers as buffers; pub use vector_common::event_test_util; pub use vector_common::{ Error, NamedInternalEvent, Result, TimeZone, assert_event_data_eq, atomic, btreemap, - byte_size_of, byte_size_of::ByteSizeOf, conversion, encode_logfmt, finalization, finalizer, id, - impl_event_data_eq, internal_event, json_size, registered_event, request_metadata, - sensitive_string, shutdown, stats, trigger, + byte_size_of, byte_size_of::ByteSizeOf, conversion, counter, encode_logfmt, finalization, + finalizer, gauge, histogram, id, impl_event_data_eq, internal_event, json_size, + registered_event, request_metadata, sensitive_string, shutdown, stats, trigger, }; pub use vector_config as configurable; pub use vector_config::impl_generate_config_from_default; diff --git a/scripts/check-events b/scripts/check-events index 5d4411b601802..18a3c61b85418 100755 --- a/scripts/check-events +++ b/scripts/check-events @@ -78,11 +78,22 @@ class Event # Scan for counter names and tags def scan_metrics(block) + # Match string literal names: counter!("metric_name", ...) block.scan(/ (counter|gauge|histogram)!\((?:\n\s+)?"([^"]+)",?(.+?)\)[;\n]/ms) \ do |type, name, tags| tags = Hash[tags.scan(/"([^"]+)" => (.+?)(?:,|$)/)] add_metric(type, name, tags) end + # Match typed enum names: counter!(CounterName::VariantName, ...) + block.scan(/ (counter|gauge|histogram)!\((?:\n\s+)?\w+Name::(\w+),?(.+?)\)[;\n]/ms) \ + do |type, variant, tags| + # Convert CamelCase variant name to snake_case metric name + name = variant.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') + .gsub(/([a-z\d])([A-Z])/, '\1_\2') + .downcase + tags = Hash[tags.scan(/"([^"]+)" => (.+?)(?:,|$)/)] + add_metric(type, name, tags) + end end # Scan the registered event macro block diff --git a/src/enrichment_tables/memory/internal_events.rs b/src/enrichment_tables/memory/internal_events.rs index 0799c6662a6af..dfd7b8d6f4c78 100644 --- a/src/enrichment_tables/memory/internal_events.rs +++ b/src/enrichment_tables/memory/internal_events.rs @@ -1,6 +1,8 @@ -use metrics::{counter, gauge}; use vector_lib::{ - NamedInternalEvent, configurable::configurable_component, internal_event::InternalEvent, + NamedInternalEvent, + configurable::configurable_component, + counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent}, }; /// Configuration of internal metrics for enrichment memory table. @@ -26,12 +28,12 @@ impl InternalEvent for MemoryEnrichmentTableRead<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_reads_total", + CounterName::MemoryEnrichmentTableReadsTotal, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_reads_total",).increment(1); + counter!(CounterName::MemoryEnrichmentTableReadsTotal,).increment(1); } } } @@ -46,12 +48,12 @@ impl InternalEvent for MemoryEnrichmentTableInserted<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_insertions_total", + CounterName::MemoryEnrichmentTableInsertionsTotal, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_insertions_total",).increment(1); + counter!(CounterName::MemoryEnrichmentTableInsertionsTotal,).increment(1); } } } @@ -64,9 +66,9 @@ pub(crate) struct MemoryEnrichmentTableFlushed { impl InternalEvent for MemoryEnrichmentTableFlushed { fn emit(self) { - counter!("memory_enrichment_table_flushes_total",).increment(1); - gauge!("memory_enrichment_table_objects_count",).set(self.new_objects_count as f64); - gauge!("memory_enrichment_table_byte_size",).set(self.new_byte_size as f64); + counter!(CounterName::MemoryEnrichmentTableFlushesTotal,).increment(1); + gauge!(GaugeName::MemoryEnrichmentTableObjectsCount,).set(self.new_objects_count as f64); + gauge!(GaugeName::MemoryEnrichmentTableByteSize,).set(self.new_byte_size as f64); } } @@ -80,12 +82,12 @@ impl InternalEvent for MemoryEnrichmentTableTtlExpired<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_ttl_expirations", + CounterName::MemoryEnrichmentTableTtlExpirations, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_ttl_expirations",).increment(1); + counter!(CounterName::MemoryEnrichmentTableTtlExpirations,).increment(1); } } } @@ -100,12 +102,12 @@ impl InternalEvent for MemoryEnrichmentTableReadFailed<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_failed_reads", + CounterName::MemoryEnrichmentTableFailedReads, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_failed_reads",).increment(1); + counter!(CounterName::MemoryEnrichmentTableFailedReads,).increment(1); } } } @@ -120,12 +122,12 @@ impl InternalEvent for MemoryEnrichmentTableInsertFailed<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_failed_insertions", + CounterName::MemoryEnrichmentTableFailedInsertions, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_failed_insertions",).increment(1); + counter!(CounterName::MemoryEnrichmentTableFailedInsertions,).increment(1); } } } diff --git a/src/internal_events/adaptive_concurrency.rs b/src/internal_events/adaptive_concurrency.rs index f815f7d4e2b4b..b43ec8452e20b 100644 --- a/src/internal_events/adaptive_concurrency.rs +++ b/src/internal_events/adaptive_concurrency.rs @@ -1,6 +1,7 @@ use std::time::Duration; -use metrics::{Histogram, histogram}; +use metrics::Histogram; +use vector_lib::{histogram, internal_event::HistogramName}; #[derive(Clone, Copy)] pub struct AdaptiveConcurrencyLimitData { @@ -17,10 +18,10 @@ registered_event! { // These are histograms, as they may have a number of different // values over each reporting interval, and each of those values // is valuable for diagnosis. - limit: Histogram = histogram!("adaptive_concurrency_limit"), - reached_limit: Histogram = histogram!("adaptive_concurrency_reached_limit"), - back_pressure: Histogram = histogram!("adaptive_concurrency_back_pressure"), - past_rtt_mean: Histogram = histogram!("adaptive_concurrency_past_rtt_mean"), + limit: Histogram = histogram!(HistogramName::AdaptiveConcurrencyLimit), + reached_limit: Histogram = histogram!(HistogramName::AdaptiveConcurrencyReachedLimit), + back_pressure: Histogram = histogram!(HistogramName::AdaptiveConcurrencyBackPressure), + past_rtt_mean: Histogram = histogram!(HistogramName::AdaptiveConcurrencyPastRttMean), } fn emit(&self, data: AdaptiveConcurrencyLimitData) { @@ -36,7 +37,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyInFlight => { - in_flight: Histogram = histogram!("adaptive_concurrency_in_flight"), + in_flight: Histogram = histogram!(HistogramName::AdaptiveConcurrencyInFlight), } fn emit(&self, in_flight: u64) { @@ -46,7 +47,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyObservedRtt => { - observed_rtt: Histogram = histogram!("adaptive_concurrency_observed_rtt"), + observed_rtt: Histogram = histogram!(HistogramName::AdaptiveConcurrencyObservedRtt), } fn emit(&self, rtt: Duration) { @@ -56,7 +57,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyAveragedRtt => { - averaged_rtt: Histogram = histogram!("adaptive_concurrency_averaged_rtt"), + averaged_rtt: Histogram = histogram!(HistogramName::AdaptiveConcurrencyAveragedRtt), } fn emit(&self, rtt: Duration) { diff --git a/src/internal_events/aggregate.rs b/src/internal_events/aggregate.rs index 1b70c4350f614..96f48f5de013c 100644 --- a/src/internal_events/aggregate.rs +++ b/src/internal_events/aggregate.rs @@ -1,13 +1,14 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AggregateEventRecorded; impl InternalEvent for AggregateEventRecorded { fn emit(self) { - counter!("aggregate_events_recorded_total").increment(1); + counter!(CounterName::AggregateEventsRecordedTotal).increment(1); } } @@ -16,7 +17,7 @@ pub struct AggregateFlushed; impl InternalEvent for AggregateFlushed { fn emit(self) { - counter!("aggregate_flushes_total").increment(1); + counter!(CounterName::AggregateFlushesTotal).increment(1); } } @@ -25,6 +26,6 @@ pub struct AggregateUpdateFailed; impl InternalEvent for AggregateUpdateFailed { fn emit(self) { - counter!("aggregate_failed_updates").increment(1); + counter!(CounterName::AggregateFailedUpdates).increment(1); } } diff --git a/src/internal_events/amqp.rs b/src/internal_events/amqp.rs index 45acdf201305a..e7864acc44e98 100644 --- a/src/internal_events/amqp.rs +++ b/src/internal_events/amqp.rs @@ -1,8 +1,9 @@ #[cfg(feature = "sources-amqp")] pub mod source { - use metrics::counter; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; + use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, + }; #[derive(Debug, NamedInternalEvent)] pub struct AmqpBytesReceived { @@ -18,7 +19,7 @@ pub mod source { protocol = %self.protocol, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol, ) .increment(self.byte_size as u64); @@ -38,7 +39,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -59,7 +60,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ) @@ -80,7 +81,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/apache_metrics.rs b/src/internal_events/apache_metrics.rs index dd5e9c34e4cf5..544dda88ba5a6 100644 --- a/src/internal_events/apache_metrics.rs +++ b/src/internal_events/apache_metrics.rs @@ -1,7 +1,6 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -19,12 +18,12 @@ impl InternalEvent for ApacheMetricsEventsReceived<'_> { fn emit(self) { trace!(message = "Events received.", count = %self.count, byte_size = %self.byte_size, endpoint = %self.endpoint); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -47,7 +46,7 @@ impl InternalEvent for ApacheMetricsParseError<'_> { endpoint = %self.endpoint, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "endpoint" => self.endpoint.to_owned(), diff --git a/src/internal_events/api.rs b/src/internal_events/api.rs index de2cb9588537d..b9196989471c8 100644 --- a/src/internal_events/api.rs +++ b/src/internal_events/api.rs @@ -1,8 +1,9 @@ use std::net::SocketAddr; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct ApiStarted { @@ -15,6 +16,6 @@ impl InternalEvent for ApiStarted { message = "API server running.", address = %self.addr, ); - counter!("api_started_total").increment(1); + counter!(CounterName::ApiStartedTotal).increment(1); } } diff --git a/src/internal_events/aws.rs b/src/internal_events/aws.rs index ff4da6c01e56b..d4c91776d7d55 100644 --- a/src/internal_events/aws.rs +++ b/src/internal_events/aws.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(NamedInternalEvent)] pub struct AwsBytesSent { @@ -21,7 +22,7 @@ impl InternalEvent for AwsBytesSent { region = ?self.region, ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "https", "region" => region, ) diff --git a/src/internal_events/aws_cloudwatch_logs.rs b/src/internal_events/aws_cloudwatch_logs.rs index 6325d6c71b25d..b4964f25d84fe 100644 --- a/src/internal_events/aws_cloudwatch_logs.rs +++ b/src/internal_events/aws_cloudwatch_logs.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct AwsCloudwatchLogsMessageSizeError { @@ -22,7 +21,7 @@ impl InternalEvent for AwsCloudwatchLogsMessageSizeError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "message_too_long", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/aws_ec2_metadata.rs b/src/internal_events/aws_ec2_metadata.rs index 43c28aee61272..c59364c75348c 100644 --- a/src/internal_events/aws_ec2_metadata.rs +++ b/src/internal_events/aws_ec2_metadata.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AwsEc2MetadataRefreshSuccessful; @@ -8,7 +9,7 @@ pub struct AwsEc2MetadataRefreshSuccessful; impl InternalEvent for AwsEc2MetadataRefreshSuccessful { fn emit(self) { debug!(message = "AWS EC2 metadata refreshed."); - counter!("metadata_refresh_successful_total").increment(1); + counter!(CounterName::MetadataRefreshSuccessfulTotal).increment(1); } } @@ -26,12 +27,12 @@ impl InternalEvent for AwsEc2MetadataRefreshError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); // deprecated - counter!("metadata_refresh_failed_total").increment(1); + counter!(CounterName::MetadataRefreshFailedTotal).increment(1); } } diff --git a/src/internal_events/aws_ecs_metrics.rs b/src/internal_events/aws_ecs_metrics.rs index 86b0d32012713..59127134e2666 100644 --- a/src/internal_events/aws_ecs_metrics.rs +++ b/src/internal_events/aws_ecs_metrics.rs @@ -1,9 +1,8 @@ use std::borrow::Cow; -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -24,12 +23,12 @@ impl InternalEvent for AwsEcsMetricsEventsReceived<'_> { endpoint = %self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_string(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_string(), ) .increment(self.byte_size.get() as u64); @@ -57,9 +56,9 @@ impl InternalEvent for AwsEcsMetricsParseError<'_> { endpoint = %self.endpoint, "Failed to parse response.", ); - counter!("parse_errors_total").increment(1); + counter!(CounterName::ParseErrorsTotal).increment(1); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "endpoint" => self.endpoint.to_string(), diff --git a/src/internal_events/aws_kinesis.rs b/src/internal_events/aws_kinesis.rs index 9fa2fb4315118..f8a7dcff49de2 100644 --- a/src/internal_events/aws_kinesis.rs +++ b/src/internal_events/aws_kinesis.rs @@ -1,8 +1,8 @@ -/// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks -use metrics::counter; use vector_lib::NamedInternalEvent; +/// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks +use vector_lib::counter; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; #[derive(Debug, NamedInternalEvent)] @@ -22,7 +22,7 @@ impl InternalEvent for AwsKinesisStreamNoPartitionKeyError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/aws_kinesis_firehose.rs b/src/internal_events/aws_kinesis_firehose.rs index 6cb4ae43ea51f..58f976839debd 100644 --- a/src/internal_events/aws_kinesis_firehose.rs +++ b/src/internal_events/aws_kinesis_firehose.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use super::prelude::{http_error_code, io_error_code}; use crate::sources::aws_kinesis_firehose::Compression; @@ -49,7 +50,7 @@ impl InternalEvent for AwsKinesisFirehoseRequestError<'_> { request_id = %self.request_id.unwrap_or(""), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, "error_code" => self.error_code, @@ -75,7 +76,7 @@ impl InternalEvent for AwsKinesisFirehoseAutomaticRecordDecodeError { compression = %self.compression, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "error_code" => io_error_code(&self.error), diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 4bdcc8ef48e6c..51e8b78d23941 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -1,11 +1,13 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::counter; #[cfg(feature = "sources-aws_s3")] pub use s3::*; +use vector_lib::counter; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs"))] -use vector_lib::internal_event::{error_stage, error_type}; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; #[cfg(feature = "sources-aws_s3")] mod s3 { @@ -15,7 +17,7 @@ mod s3 { BatchResultErrorEntry, DeleteMessageBatchRequestEntry, DeleteMessageBatchResultEntry, SendMessageBatchRequestEntry, SendMessageBatchResultEntry, }; - use metrics::histogram; + use vector_lib::histogram; use super::*; use crate::sources::aws_s3::sqs::ProcessingError; @@ -34,7 +36,7 @@ mod s3 { duration_ms = %self.duration.as_millis(), ); histogram!( - "s3_object_processing_succeeded_duration_seconds", + HistogramName::S3ObjectProcessingSucceededDurationSeconds, "bucket" => self.bucket.to_owned(), ) .record(self.duration); @@ -55,7 +57,7 @@ mod s3 { duration_ms = %self.duration.as_millis(), ); histogram!( - "s3_object_processing_failed_duration_seconds", + HistogramName::S3ObjectProcessingFailedDurationSeconds, "bucket" => self.bucket.to_owned(), ) .record(self.duration); @@ -79,7 +81,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_processing_sqs_message", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -100,7 +102,8 @@ mod s3 { .map(|x| x.id.as_str()) .collect::>() .join(", ")); - counter!("sqs_message_delete_succeeded_total").increment(self.message_ids.len() as u64); + counter!(CounterName::SqsMessageDeleteSucceededTotal) + .increment(self.message_ids.len() as u64); } } @@ -122,7 +125,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deleting_some_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -151,7 +154,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deleting_all_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -172,7 +175,8 @@ mod s3 { .map(|x| x.id.as_str()) .collect::>() .join(", ")); - counter!("sqs_message_defer_succeeded_total").increment(self.message_ids.len() as u64); + counter!(CounterName::SqsMessageDeferSucceededTotal) + .increment(self.message_ids.len() as u64); } } @@ -194,7 +198,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deferring_some_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -223,7 +227,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deferring_all_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -248,7 +252,7 @@ impl InternalEvent for SqsMessageReceiveError<'_, E> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_fetching_sqs_events", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -265,8 +269,8 @@ pub struct SqsMessageReceiveSucceeded { impl InternalEvent for SqsMessageReceiveSucceeded { fn emit(self) { trace!(message = "Received SQS messages.", count = %self.count); - counter!("sqs_message_receive_succeeded_total").increment(1); - counter!("sqs_message_received_messages_total").increment(self.count as u64); + counter!(CounterName::SqsMessageReceiveSucceededTotal).increment(1); + counter!(CounterName::SqsMessageReceivedMessagesTotal).increment(self.count as u64); } } @@ -278,7 +282,7 @@ pub struct SqsMessageProcessingSucceeded<'a> { impl InternalEvent for SqsMessageProcessingSucceeded<'_> { fn emit(self) { trace!(message = "Processed SQS message successfully.", message_id = %self.message_id); - counter!("sqs_message_processing_succeeded_total").increment(1); + counter!(CounterName::SqsMessageProcessingSucceededTotal).increment(1); } } @@ -300,7 +304,7 @@ impl InternalEvent for SqsMessageDeleteError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -322,7 +326,7 @@ impl InternalEvent for SqsS3EventRecordInvalidEventIgnored<'_> { fn emit(self) { warn!(message = "Ignored S3 record in SQS message for an event that was not ObjectCreated.", bucket = %self.bucket, key = %self.key, kind = %self.kind, name = %self.name); - counter!("sqs_s3_event_record_ignored_total", "ignore_type" => "invalid_event_kind") + counter!(CounterName::SqsS3EventRecordIgnoredTotal, "ignore_type" => "invalid_event_kind") .increment(1); } } diff --git a/src/internal_events/batch.rs b/src/internal_events/batch.rs index 8b7a4decfa770..bd553c1e9863d 100644 --- a/src/internal_events/batch.rs +++ b/src/internal_events/batch.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct LargeEventDroppedError { @@ -21,7 +20,7 @@ impl InternalEvent for LargeEventDroppedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "oversized", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/common.rs b/src/internal_events/common.rs index 69877e3db9ef5..5a6f9c50ede93 100644 --- a/src/internal_events/common.rs +++ b/src/internal_events/common.rs @@ -1,11 +1,12 @@ use std::time::Instant; -use metrics::{counter, histogram}; use vector_lib::NamedInternalEvent; pub use vector_lib::internal_event::EventsReceived; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{counter, histogram}; #[derive(Debug, NamedInternalEvent)] pub struct EndpointBytesReceived<'a> { @@ -23,7 +24,7 @@ impl InternalEvent for EndpointBytesReceived<'_> { endpoint = %self.endpoint, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol.to_owned(), "endpoint" => self.endpoint.to_owned(), ) @@ -47,7 +48,7 @@ impl InternalEvent for EndpointBytesSent<'_> { endpoint = %self.endpoint ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => self.protocol.to_string(), "endpoint" => self.endpoint.to_string() ) @@ -70,7 +71,7 @@ impl InternalEvent for SocketOutgoingConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_connecting", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::SENDING, @@ -95,7 +96,7 @@ impl InternalEvent for StreamClosedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => STREAM_CLOSED, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -117,8 +118,8 @@ pub struct CollectionCompleted { impl InternalEvent for CollectionCompleted { fn emit(self) { debug!(message = "Collection completed."); - counter!("collect_completed_total").increment(1); - histogram!("collect_duration_seconds").record(self.end - self.start); + counter!(CounterName::CollectCompletedTotal).increment(1); + histogram!(HistogramName::CollectDurationSeconds).record(self.end - self.start); } } @@ -139,7 +140,7 @@ impl InternalEvent for SinkRequestBuildError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/conditions.rs b/src/internal_events/conditions.rs index 96f7ae0ae0d50..e89f63b6a9c54 100644 --- a/src/internal_events/conditions.rs +++ b/src/internal_events/conditions.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, Copy, Clone, NamedInternalEvent)] pub struct VrlConditionExecutionError<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for VrlConditionExecutionError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/datadog_agent.rs b/src/internal_events/datadog_agent.rs index 9ee5296407b24..84a4a8535c0c7 100644 --- a/src/internal_events/datadog_agent.rs +++ b/src/internal_events/datadog_agent.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogAgentJsonParseError<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for DatadogAgentJsonParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 5daa3ab87fa79..00e7d50a8a7d7 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogMetricsEncodingError<'a> { @@ -21,7 +20,7 @@ impl InternalEvent for DatadogMetricsEncodingError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/datadog_traces.rs b/src/internal_events/datadog_traces.rs index 06b48552fb840..016002681f81b 100644 --- a/src/internal_events/datadog_traces.rs +++ b/src/internal_events/datadog_traces.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogTracesEncodingError { @@ -22,7 +21,7 @@ impl InternalEvent for DatadogTracesEncodingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -51,7 +50,7 @@ impl InternalEvent for DatadogTracesAPMStatsError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/src/internal_events/dedupe.rs b/src/internal_events/dedupe.rs index eb7511578cdd7..a1c36a3c4fa2b 100644 --- a/src/internal_events/dedupe.rs +++ b/src/internal_events/dedupe.rs @@ -1,5 +1,7 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DedupeEventsDropped { diff --git a/src/internal_events/demo_logs.rs b/src/internal_events/demo_logs.rs index 406a9856fe6e9..3958f41421ca7 100644 --- a/src/internal_events/demo_logs.rs +++ b/src/internal_events/demo_logs.rs @@ -1,5 +1,4 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; #[derive(Debug, NamedInternalEvent)] pub struct DemoLogsEventProcessed; diff --git a/src/internal_events/dnstap.rs b/src/internal_events/dnstap.rs index 2fc20b95848d1..1ef67ee3001e0 100644 --- a/src/internal_events/dnstap.rs +++ b/src/internal_events/dnstap.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub(crate) struct DnstapParseError { @@ -16,7 +17,7 @@ impl InternalEvent for DnstapParseError { error_type = error_type::PARSER_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) diff --git a/src/internal_events/docker_logs.rs b/src/internal_events/docker_logs.rs index bb44e112e05b8..fe24d41b6350b 100644 --- a/src/internal_events/docker_logs.rs +++ b/src/internal_events/docker_logs.rs @@ -1,9 +1,8 @@ use bollard::errors::Error; use chrono::ParseError; -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,11 +22,11 @@ impl InternalEvent for DockerLogsEventsReceived<'_> { container_id = %self.container_id ); counter!( - "component_received_events_total", "container_name" => self.container_name.to_owned() + CounterName::ComponentReceivedEventsTotal, "container_name" => self.container_name.to_owned() ) .increment(1); counter!( - "component_received_event_bytes_total", "container_name" => self.container_name.to_owned() + CounterName::ComponentReceivedEventBytesTotal, "container_name" => self.container_name.to_owned() ).increment(self.byte_size.get() as u64); } } @@ -45,7 +44,7 @@ impl InternalEvent for DockerLogsContainerEventReceived<'_> { container_id = %self.container_id, action = %self.action, ); - counter!("container_processed_events_total").increment(1); + counter!(CounterName::ContainerProcessedEventsTotal).increment(1); } } @@ -60,7 +59,7 @@ impl InternalEvent for DockerLogsContainerWatch<'_> { message = "Started watching for container logs.", container_id = %self.container_id, ); - counter!("containers_watched_total").increment(1); + counter!(CounterName::ContainersWatchedTotal).increment(1); } } @@ -75,7 +74,7 @@ impl InternalEvent for DockerLogsContainerUnwatch<'_> { message = "Stopped watching for container logs.", container_id = %self.container_id, ); - counter!("containers_unwatched_total").increment(1); + counter!(CounterName::ContainersUnwatchedTotal).increment(1); } } @@ -95,7 +94,7 @@ impl InternalEvent for DockerLogsCommunicationError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) @@ -119,7 +118,7 @@ impl InternalEvent for DockerLogsContainerMetadataFetchError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, "container_id" => self.container_id.to_owned(), @@ -144,7 +143,7 @@ impl InternalEvent for DockerLogsTimestampParseError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "container_id" => self.container_id.to_owned(), @@ -169,7 +168,7 @@ impl InternalEvent for DockerLogsLoggingDriverUnsupportedError<'_> { container_id = ?self.container_id, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::RECEIVING, "container_id" => self.container_id.to_owned(), diff --git a/src/internal_events/doris.rs b/src/internal_events/doris.rs index 0a2430869b7c5..15dffd26418b4 100644 --- a/src/internal_events/doris.rs +++ b/src/internal_events/doris.rs @@ -1,5 +1,7 @@ -use metrics::counter; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; /// Emitted when rows are successfully loaded into Doris. #[derive(Debug, NamedInternalEvent)] @@ -17,10 +19,10 @@ impl InternalEvent for DorisRowsLoaded { ); // Record the number of rows loaded - counter!("doris_rows_loaded_total").increment(self.loaded_rows as u64); + counter!(CounterName::DorisRowsLoadedTotal).increment(self.loaded_rows as u64); // Record the number of bytes loaded - counter!("doris_bytes_loaded_total").increment(self.load_bytes as u64); + counter!(CounterName::DorisBytesLoadedTotal).increment(self.load_bytes as u64); } } @@ -37,6 +39,6 @@ impl InternalEvent for DorisRowsFiltered { filtered_rows = %self.filtered_rows ); - counter!("doris_rows_filtered_total").increment(self.filtered_rows as u64); + counter!(CounterName::DorisRowsFilteredTotal).increment(self.filtered_rows as u64); } } diff --git a/src/internal_events/encoding_transcode.rs b/src/internal_events/encoding_transcode.rs index 76ad81792a63a..ec6280f1c703d 100644 --- a/src/internal_events/encoding_transcode.rs +++ b/src/internal_events/encoding_transcode.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DecoderBomRemoval { @@ -13,7 +14,7 @@ impl InternalEvent for DecoderBomRemoval { message = "Removing initial BOM bytes from the final output while decoding to utf8.", from_encoding = %self.from_encoding ); - counter!("decoder_bom_removals_total").increment(1); + counter!(CounterName::DecoderBomRemovalsTotal).increment(1); } } @@ -30,7 +31,7 @@ impl InternalEvent for DecoderMalformedReplacement { ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) - counter!("decoder_malformed_replacement_warnings_total").increment(1); + counter!(CounterName::DecoderMalformedReplacementWarningsTotal).increment(1); } } @@ -47,6 +48,6 @@ impl InternalEvent for EncoderUnmappableReplacement { ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) - counter!("encoder_unmappable_replacement_warnings_total").increment(1); + counter!(CounterName::EncoderUnmappableReplacementWarningsTotal).increment(1); } } diff --git a/src/internal_events/eventstoredb_metrics.rs b/src/internal_events/eventstoredb_metrics.rs index c72b9c7dbeb7c..c7db7fad354a5 100644 --- a/src/internal_events/eventstoredb_metrics.rs +++ b/src/internal_events/eventstoredb_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct EventStoreDbMetricsHttpError { @@ -16,7 +17,7 @@ impl InternalEvent for EventStoreDbMetricsHttpError { error_type = error_type::REQUEST_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, ) @@ -38,7 +39,7 @@ impl InternalEvent for EventStoreDbStatsParsingError { error_type = error_type::PARSER_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) diff --git a/src/internal_events/exec.rs b/src/internal_events/exec.rs index abe226897d5d3..21edc68f9df2a 100644 --- a/src/internal_events/exec.rs +++ b/src/internal_events/exec.rs @@ -1,11 +1,11 @@ use std::time::Duration; -use metrics::{counter, histogram}; use tokio::time::error::Elapsed; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, histogram, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -28,12 +28,12 @@ impl InternalEvent for ExecEventsReceived<'_> { command = %self.command, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "command" => self.command.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "command" => self.command.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -57,7 +57,7 @@ impl InternalEvent for ExecFailedError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => self.command.to_owned(), "error_type" => error_type::COMMAND_FAILED, "error_code" => io_error_code(&self.error), @@ -85,7 +85,7 @@ impl InternalEvent for ExecTimeoutError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => self.command.to_owned(), "error_type" => error_type::TIMED_OUT, "stage" => error_stage::RECEIVING, @@ -120,14 +120,14 @@ impl InternalEvent for ExecCommandExecuted<'_> { elapsed_millis = %self.exec_duration.as_millis(), ); counter!( - "command_executed_total", + CounterName::CommandExecutedTotal, "command" => self.command.to_owned(), "exit_status" => exit_status.clone(), ) .increment(1); histogram!( - "command_execution_duration_seconds", + HistogramName::CommandExecutionDurationSeconds, "exit_status" => exit_status, "command" => self.command.to_owned(), ) @@ -196,7 +196,7 @@ impl InternalEvent for ExecFailedToSignalChildError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => format!("{:?}", self.command.as_std()), "error_code" => self.error.to_error_code(), "error_type" => error_type::COMMAND_FAILED, @@ -218,7 +218,7 @@ impl InternalEvent for ExecChannelClosedError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/expansion.rs b/src/internal_events/expansion.rs index 7b2633dd64f97..7f5b48914c69c 100644 --- a/src/internal_events/expansion.rs +++ b/src/internal_events/expansion.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(NamedInternalEvent)] pub struct PairExpansionError<'a> { @@ -25,7 +24,7 @@ impl InternalEvent for PairExpansionError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index 0e85c67150621..f1299ae751694 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -2,12 +2,13 @@ use std::borrow::Cow; -use metrics::{counter, gauge}; use vector_lib::{ NamedInternalEvent, configurable::configurable_component, + counter, gauge, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }, }; @@ -34,7 +35,7 @@ pub struct FileOpen { impl InternalEvent for FileOpen { fn emit(self) { - gauge!("open_files").set(self.count as f64); + gauge!(GaugeName::OpenFiles).set(self.count as f64); } } @@ -55,13 +56,13 @@ impl InternalEvent for FileBytesSent<'_> { ); if self.include_file_metric_tag { counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "file", "file" => self.file.clone().into_owned(), ) } else { counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "file", ) } @@ -89,7 +90,7 @@ impl InternalEvent for FileIoError<'_, P> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.code, "error_type" => error_type::IO_FAILED, "stage" => error_stage::SENDING, @@ -110,11 +111,12 @@ mod source { use std::{io::Error, path::Path, time::Duration}; use bytes::BytesMut; - use metrics::counter; use vector_lib::{ - NamedInternalEvent, emit, + NamedInternalEvent, counter, emit, file_source_common::internal_events::FileSourceInternalEvents, - internal_event::{ComponentEventsDropped, INTENTIONAL, error_stage, error_type}, + internal_event::{ + ComponentEventsDropped, CounterName, INTENTIONAL, error_stage, error_type, + }, json_size::JsonSize, }; @@ -137,13 +139,13 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "file", "file" => self.file.to_owned() ) } else { counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "file", ) } @@ -169,18 +171,18 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "file" => self.file.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "file" => self.file.to_owned(), ) .increment(self.byte_size.get() as u64); } else { - counter!("component_received_events_total").increment(self.count as u64); - counter!("component_received_event_bytes_total") + counter!(CounterName::ComponentReceivedEventsTotal).increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal) .increment(self.byte_size.get() as u64); } } @@ -200,11 +202,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "checksum_errors_total", + CounterName::ChecksumErrorsTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("checksum_errors_total") + counter!(CounterName::ChecksumErrorsTotal) } .increment(1); } @@ -229,7 +231,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_fingerprint", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -237,7 +239,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_fingerprint", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -268,7 +270,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "file" => self.file.to_string_lossy().into_owned(), "error_code" => DELETION_FAILED, "error_type" => error_type::COMMAND_FAILED, @@ -276,7 +278,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => DELETION_FAILED, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -300,11 +302,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_deleted_total", + CounterName::FilesDeletedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_deleted_total") + counter!(CounterName::FilesDeletedTotal) } .increment(1); } @@ -327,13 +329,13 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_unwatched_total", + CounterName::FilesUnwatchedTotal, "file" => self.file.to_string_lossy().into_owned(), "reached_eof" => reached_eof, ) } else { counter!( - "files_unwatched_total", + CounterName::FilesUnwatchedTotal, "reached_eof" => reached_eof, ) } @@ -360,7 +362,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "watching", "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -368,7 +370,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "watching", "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -394,11 +396,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_resumed_total", + CounterName::FilesResumedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_resumed_total") + counter!(CounterName::FilesResumedTotal) } .increment(1); } @@ -418,11 +420,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_added_total", + CounterName::FilesAddedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_added_total") + counter!(CounterName::FilesAddedTotal) } .increment(1); } @@ -441,7 +443,7 @@ mod source { count = %self.count, duration_ms = self.duration.as_millis() as u64, ); - counter!("checkpoints_total").increment(self.count as u64); + counter!(CounterName::CheckpointsTotal).increment(self.count as u64); } } @@ -460,7 +462,7 @@ mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "writing_checkpoints", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::RECEIVING, @@ -486,7 +488,7 @@ mod source { path = %self.path.display(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "globbing", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -513,7 +515,7 @@ mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_line_from_file", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/file_descriptor.rs b/src/internal_events/file_descriptor.rs index 7882ed4d95b12..8db4e7895729e 100644 --- a/src/internal_events/file_descriptor.rs +++ b/src/internal_events/file_descriptor.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct FileDescriptorReadError { @@ -19,7 +20,7 @@ where stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/fluent.rs b/src/internal_events/fluent.rs index 6d6bb408edc5f..f47b927d5cbe2 100644 --- a/src/internal_events/fluent.rs +++ b/src/internal_events/fluent.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::sources::fluent::DecodeError; @@ -31,7 +32,7 @@ impl InternalEvent for FluentMessageDecodeError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/gcp_pubsub.rs b/src/internal_events/gcp_pubsub.rs index 4066d71936529..752546cfa7d09 100644 --- a/src/internal_events/gcp_pubsub.rs +++ b/src/internal_events/gcp_pubsub.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(NamedInternalEvent)] pub struct GcpPubsubConnectError { @@ -18,7 +19,7 @@ impl InternalEvent for GcpPubsubConnectError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_connecting", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, @@ -43,7 +44,7 @@ impl InternalEvent for GcpPubsubStreamingPullError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_streaming_pull", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -68,7 +69,7 @@ impl InternalEvent for GcpPubsubReceiveError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_fetching_events", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/grpc.rs b/src/internal_events/grpc.rs index 882b1dbfaf33b..66da41e10b797 100644 --- a/src/internal_events/grpc.rs +++ b/src/internal_events/grpc.rs @@ -1,10 +1,11 @@ use std::time::Duration; use http::response::Response; -use metrics::{counter, histogram}; use tonic::Code; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; const GRPC_STATUS_LABEL: &str = "grpc_status"; @@ -13,7 +14,7 @@ pub struct GrpcServerRequestReceived; impl InternalEvent for GrpcServerRequestReceived { fn emit(self) { - counter!("grpc_server_messages_received_total").increment(1); + counter!(CounterName::GrpcServerMessagesReceivedTotal).increment(1); } } @@ -34,8 +35,8 @@ impl InternalEvent for GrpcServerResponseSent<'_, B> { let grpc_code = grpc_code_to_name(grpc_code); let labels = &[(GRPC_STATUS_LABEL, grpc_code)]; - counter!("grpc_server_messages_sent_total", labels).increment(1); - histogram!("grpc_server_handler_duration_seconds", labels).record(self.latency); + counter!(CounterName::GrpcServerMessagesSentTotal, labels).increment(1); + histogram!(HistogramName::GrpcServerHandlerDurationSeconds, labels).record(self.latency); } } @@ -53,7 +54,7 @@ impl InternalEvent for GrpcInvalidCompressionSchemeError<'_> { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -78,7 +79,7 @@ where stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/heartbeat.rs b/src/internal_events/heartbeat.rs index 658a24fcb8762..16191669e7317 100644 --- a/src/internal_events/heartbeat.rs +++ b/src/internal_events/heartbeat.rs @@ -1,8 +1,9 @@ use std::time::Instant; -use metrics::gauge; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, gauge, + internal_event::{GaugeName, InternalEvent}, +}; use crate::built_info; @@ -14,9 +15,9 @@ pub struct Heartbeat { impl InternalEvent for Heartbeat { fn emit(self) { trace!(target: "vector", message = "Beep."); - gauge!("uptime_seconds").set(self.since.elapsed().as_secs() as f64); + gauge!(GaugeName::UptimeSeconds).set(self.since.elapsed().as_secs() as f64); gauge!( - "build_info", + GaugeName::BuildInfo, "debug" => built_info::DEBUG, "version" => built_info::PKG_VERSION, "rust_version" => built_info::RUST_VERSION, diff --git a/src/internal_events/host_metrics.rs b/src/internal_events/host_metrics.rs index d60771644571e..7662ca5e808a4 100644 --- a/src/internal_events/host_metrics.rs +++ b/src/internal_events/host_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct HostMetricsScrapeError { @@ -16,7 +17,7 @@ impl InternalEvent for HostMetricsScrapeError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) @@ -40,7 +41,7 @@ impl InternalEvent for HostMetricsScrapeDetailError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) @@ -66,7 +67,7 @@ impl InternalEvent for HostMetricsScrapeFilesystemError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/http.rs b/src/internal_events/http.rs index a0d8f5d2a92d3..4aad267d7683b 100644 --- a/src/internal_events/http.rs +++ b/src/internal_events/http.rs @@ -1,10 +1,9 @@ use std::{error::Error, time::Duration}; use http::Response; -use metrics::{counter, histogram}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -16,7 +15,7 @@ pub struct HttpServerRequestReceived; impl InternalEvent for HttpServerRequestReceived { fn emit(self) { debug!(message = "Received HTTP request."); - counter!("http_server_requests_received_total").increment(1); + counter!(CounterName::HttpServerRequestsReceivedTotal).increment(1); } } @@ -32,8 +31,8 @@ impl InternalEvent for HttpServerResponseSent<'_, B> { HTTP_STATUS_LABEL, self.response.status().as_u16().to_string(), )]; - counter!("http_server_responses_sent_total", labels).increment(1); - histogram!("http_server_handler_duration_seconds", labels).record(self.latency); + counter!(CounterName::HttpServerResponsesSentTotal, labels).increment(1); + histogram!(HistogramName::HttpServerHandlerDurationSeconds, labels).record(self.latency); } } @@ -53,7 +52,7 @@ impl InternalEvent for HttpBytesReceived<'_> { protocol = %self.protocol ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) @@ -79,15 +78,15 @@ impl InternalEvent for HttpEventsReceived<'_> { protocol = %self.protocol, ); - histogram!("component_received_events_count").record(self.count as f64); + histogram!(HistogramName::ComponentReceivedEventsCount).record(self.count as f64); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) @@ -126,7 +125,7 @@ impl InternalEvent for HttpBadRequest<'_> { http_code = %self.code, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::REQUEST_FAILED, "error_stage" => error_stage::RECEIVING, @@ -152,7 +151,7 @@ impl InternalEvent for HttpDecompressError<'_> { encoding = %self.encoding ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_decompressing_payload", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, @@ -174,7 +173,7 @@ impl InternalEvent for HttpInternalError<'_> { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index d1e4c7549d15a..346192dfca141 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -5,9 +5,10 @@ use http::{ header::{self, HeaderMap, HeaderValue}, }; use hyper::{Error, body::HttpBody}; -use metrics::{counter, histogram}; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AboutToSendHttpRequest<'a, T> { @@ -39,7 +40,7 @@ impl InternalEvent for AboutToSendHttpRequest<'_, T> { headers = ?remove_sensitive(self.request.headers()), body = %FormatBody(self.request.body()), ); - counter!("http_client_requests_sent_total", "method" => self.request.method().to_string()) + counter!(CounterName::HttpClientRequestsSentTotal, "method" => self.request.method().to_string()) .increment(1); } } @@ -60,13 +61,13 @@ impl InternalEvent for GotHttpResponse<'_, T> { body = %FormatBody(self.response.body()), ); counter!( - "http_client_responses_total", + CounterName::HttpClientResponsesTotal, "status" => self.response.status().as_u16().to_string(), ) .increment(1); - histogram!("http_client_rtt_seconds").record(self.roundtrip); + histogram!(HistogramName::HttpClientRttSeconds).record(self.roundtrip); histogram!( - "http_client_response_rtt_seconds", + HistogramName::HttpClientResponseRttSeconds, "status" => self.response.status().as_u16().to_string(), ) .record(self.roundtrip); @@ -87,9 +88,10 @@ impl InternalEvent for GotHttpWarning<'_> { error_type = error_type::REQUEST_FAILED, stage = error_stage::PROCESSING, ); - counter!("http_client_errors_total", "error_kind" => self.error.to_string()).increment(1); - histogram!("http_client_rtt_seconds").record(self.roundtrip); - histogram!("http_client_error_rtt_seconds", "error_kind" => self.error.to_string()) + counter!(CounterName::HttpClientErrorsTotal, "error_kind" => self.error.to_string()) + .increment(1); + histogram!(HistogramName::HttpClientRttSeconds).record(self.roundtrip); + histogram!(HistogramName::HttpClientErrorRttSeconds, "error_kind" => self.error.to_string()) .record(self.roundtrip); } } diff --git a/src/internal_events/http_client_source.rs b/src/internal_events/http_client_source.rs index ff67535c0c14b..0ba52d722dfee 100644 --- a/src/internal_events/http_client_source.rs +++ b/src/internal_events/http_client_source.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -25,12 +24,12 @@ impl InternalEvent for HttpClientEventsReceived { url = %self.url, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "uri" => self.url.clone(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "uri" => self.url.clone(), ) .increment(self.byte_size.get() as u64); @@ -53,7 +52,7 @@ impl InternalEvent for HttpClientHttpResponseError { error_code = %http_error_code(self.code.as_u16()), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "url" => self.url, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, @@ -79,7 +78,7 @@ impl InternalEvent for HttpClientHttpError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "url" => self.url, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/influxdb.rs b/src/internal_events/influxdb.rs index 22b34c3dc532c..a142642140f6f 100644 --- a/src/internal_events/influxdb.rs +++ b/src/internal_events/influxdb.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct InfluxdbEncodingError { @@ -20,7 +19,7 @@ impl InternalEvent for InfluxdbEncodingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/internal_logs.rs b/src/internal_events/internal_logs.rs index 63c520723c9c7..2f0674d380edd 100644 --- a/src/internal_events/internal_logs.rs +++ b/src/internal_events/internal_logs.rs @@ -1,5 +1,8 @@ -use metrics::counter; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent, json_size::JsonSize}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, + json_size::JsonSize, +}; #[derive(Debug, NamedInternalEvent)] pub struct InternalLogsBytesReceived { @@ -10,7 +13,7 @@ impl InternalEvent for InternalLogsBytesReceived { fn emit(self) { // MUST NOT emit logs here to avoid an infinite log loop counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "internal", ) .increment(self.byte_size as u64); @@ -26,7 +29,8 @@ pub struct InternalLogsEventsReceived { impl InternalEvent for InternalLogsEventsReceived { fn emit(self) { // MUST NOT emit logs here to avoid an infinite log loop - counter!("component_received_events_total").increment(self.count as u64); - counter!("component_received_event_bytes_total").increment(self.byte_size.get() as u64); + counter!(CounterName::ComponentReceivedEventsTotal).increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal) + .increment(self.byte_size.get() as u64); } } diff --git a/src/internal_events/journald.rs b/src/internal_events/journald.rs index 0debba0de5d29..02f6300cb6905 100644 --- a/src/internal_events/journald.rs +++ b/src/internal_events/journald.rs @@ -1,8 +1,8 @@ -use metrics::counter; use vector_lib::{ NamedInternalEvent, codecs::decoding::BoxedFramingError, - internal_event::{InternalEvent, error_stage, error_type}, + counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[derive(Debug, NamedInternalEvent)] @@ -21,7 +21,7 @@ impl InternalEvent for JournaldInvalidRecordError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) @@ -43,7 +43,7 @@ impl InternalEvent for JournaldStartJournalctlError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::COMMAND_FAILED, ) @@ -65,7 +65,7 @@ impl InternalEvent for JournaldReadError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::READER_FAILED, ) @@ -89,7 +89,7 @@ impl InternalEvent for JournaldCheckpointSetError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::IO_FAILED, ) @@ -113,7 +113,7 @@ impl InternalEvent for JournaldCheckpointFileOpenError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::IO_FAILED, ) diff --git a/src/internal_events/kafka.rs b/src/internal_events/kafka.rs index 87ad581e1c453..1cbd2ba3ce8cb 100644 --- a/src/internal_events/kafka.rs +++ b/src/internal_events/kafka.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::{counter, gauge}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; use vrl::path::OwnedTargetPath; @@ -26,7 +25,7 @@ impl InternalEvent for KafkaBytesReceived<'_> { partition = %self.partition, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), @@ -53,13 +52,13 @@ impl InternalEvent for KafkaEventsReceived<'_> { partition = %self.partition, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), ) @@ -82,7 +81,7 @@ impl InternalEvent for KafkaOffsetUpdateError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "kafka_offset_update", "error_type" => error_type::READER_FAILED, "stage" => error_stage::SENDING, @@ -106,7 +105,7 @@ impl InternalEvent for KafkaReadError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_message", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -123,24 +122,24 @@ pub struct KafkaStatisticsReceived<'a> { impl InternalEvent for KafkaStatisticsReceived<'_> { fn emit(self) { - gauge!("kafka_queue_messages").set(self.statistics.msg_cnt as f64); - gauge!("kafka_queue_messages_bytes").set(self.statistics.msg_size as f64); - counter!("kafka_requests_total").absolute(self.statistics.tx as u64); - counter!("kafka_requests_bytes_total").absolute(self.statistics.tx_bytes as u64); - counter!("kafka_responses_total").absolute(self.statistics.rx as u64); - counter!("kafka_responses_bytes_total").absolute(self.statistics.rx_bytes as u64); - counter!("kafka_produced_messages_total").absolute(self.statistics.txmsgs as u64); - counter!("kafka_produced_messages_bytes_total") + gauge!(GaugeName::KafkaQueueMessages).set(self.statistics.msg_cnt as f64); + gauge!(GaugeName::KafkaQueueMessagesBytes).set(self.statistics.msg_size as f64); + counter!(CounterName::KafkaRequestsTotal).absolute(self.statistics.tx as u64); + counter!(CounterName::KafkaRequestsBytesTotal).absolute(self.statistics.tx_bytes as u64); + counter!(CounterName::KafkaResponsesTotal).absolute(self.statistics.rx as u64); + counter!(CounterName::KafkaResponsesBytesTotal).absolute(self.statistics.rx_bytes as u64); + counter!(CounterName::KafkaProducedMessagesTotal).absolute(self.statistics.txmsgs as u64); + counter!(CounterName::KafkaProducedMessagesBytesTotal) .absolute(self.statistics.txmsg_bytes as u64); - counter!("kafka_consumed_messages_total").absolute(self.statistics.rxmsgs as u64); - counter!("kafka_consumed_messages_bytes_total") + counter!(CounterName::KafkaConsumedMessagesTotal).absolute(self.statistics.rxmsgs as u64); + counter!(CounterName::KafkaConsumedMessagesBytesTotal) .absolute(self.statistics.rxmsg_bytes as u64); if self.expose_lag_metrics { for (topic_id, topic) in &self.statistics.topics { for (partition_id, partition) in &topic.partitions { gauge!( - "kafka_consumer_lag", + GaugeName::KafkaConsumerLag, "topic_id" => topic_id.clone(), "partition_id" => partition_id.to_string(), ) @@ -166,7 +165,7 @@ impl InternalEvent for KafkaHeaderExtractionError<'_> { header_field = self.header_field.to_string(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "extracting_header", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index c6edb03724169..15b4ca4e61ee0 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,8 +1,8 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -37,21 +37,21 @@ impl InternalEvent for KubernetesLogsEventsReceived<'_> { let pod_namespace = pod_info.namespace; counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "pod_name" => pod_name.clone(), "pod_namespace" => pod_namespace.clone(), ) .increment(1); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "pod_name" => pod_name, "pod_namespace" => pod_namespace, ) .increment(self.byte_size.get() as u64); } None => { - counter!("component_received_events_total").increment(1); - counter!("component_received_event_bytes_total") + counter!(CounterName::ComponentReceivedEventsTotal).increment(1); + counter!(CounterName::ComponentReceivedEventBytesTotal) .increment(self.byte_size.get() as u64); } } @@ -75,7 +75,7 @@ impl InternalEvent for KubernetesLogsEventAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, @@ -99,13 +99,13 @@ impl InternalEvent for KubernetesLogsEventNamespaceAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_event_namespace_annotation_failures_total").increment(1); + counter!(CounterName::K8sEventNamespaceAnnotationFailuresTotal).increment(1); } } @@ -124,13 +124,13 @@ impl InternalEvent for KubernetesLogsEventNodeAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_event_node_annotation_failures_total").increment(1); + counter!(CounterName::K8sEventNodeAnnotationFailuresTotal).increment(1); } } @@ -145,7 +145,7 @@ impl InternalEvent for KubernetesLogsFormatPickerEdgeCase { message = "Encountered format picker edge case.", what = %self.what, ); - counter!("k8s_format_picker_edge_cases_total").increment(1); + counter!(CounterName::K8sFormatPickerEdgeCasesTotal).increment(1); } } @@ -163,12 +163,12 @@ impl InternalEvent for KubernetesLogsDockerFormatParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_docker_format_parse_failures_total").increment(1); + counter!(CounterName::K8sDockerFormatParseFailuresTotal).increment(1); } } @@ -191,7 +191,7 @@ impl InternalEvent for KubernetesLifecycleError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => KUBERNETES_LIFECYCLE, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, @@ -222,7 +222,7 @@ impl InternalEvent for KubernetesMergedLineTooBigError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_line_from_kubernetes_log", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/log_to_metric.rs b/src/internal_events/log_to_metric.rs index 9fbb64dba0622..e8e3da5b898b7 100644 --- a/src/internal_events/log_to_metric.rs +++ b/src/internal_events/log_to_metric.rs @@ -1,9 +1,10 @@ use std::num::ParseFloatError; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; #[derive(NamedInternalEvent)] @@ -22,7 +23,7 @@ impl InternalEvent for LogToMetricFieldNullError<'_> { null_field = %self.field ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "field_null", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -52,7 +53,7 @@ impl InternalEvent for LogToMetricParseFloatError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_parsing_float", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -83,7 +84,7 @@ impl InternalEvent for MetricMetadataInvalidFieldValueError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_field_value", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -112,7 +113,7 @@ impl InternalEvent for MetricMetadataParseError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => format!("failed_parsing_{}", self.kind), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -137,7 +138,7 @@ impl InternalEvent for MetricMetadataMetricDetailsNotFoundError { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "missing_metric_details", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/logplex.rs b/src/internal_events/logplex.rs index b28588c5c2d1e..4a45df6349b65 100644 --- a/src/internal_events/logplex.rs +++ b/src/internal_events/logplex.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use super::prelude::io_error_code; @@ -37,7 +38,7 @@ impl InternalEvent for HerokuLogplexRequestReadError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "error_code" => io_error_code(&self.error), "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/loki.rs b/src/internal_events/loki.rs index 523efd7e4f59b..3a5935e1a372a 100644 --- a/src/internal_events/loki.rs +++ b/src/internal_events/loki.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct LokiEventUnlabeledError; @@ -17,7 +16,7 @@ impl InternalEvent for LokiEventUnlabeledError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "unlabeled_event", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -48,7 +47,7 @@ impl InternalEvent for LokiOutOfOrderEventDroppedError { }); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "out_of_order", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -69,7 +68,7 @@ impl InternalEvent for LokiOutOfOrderEventRewritten { count = self.count, reason = "out_of_order", ); - counter!("rewritten_timestamp_events_total").increment(self.count as u64); + counter!(CounterName::RewrittenTimestampEventsTotal).increment(self.count as u64); } } @@ -90,7 +89,7 @@ impl InternalEvent for LokiTimestampNonParsableEventsDropped { emit!(ComponentEventsDropped:: { count: 1, reason }); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "non-parsable_timestamp", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/lua.rs b/src/internal_events/lua.rs index feb9d04212b88..3a8779d9cd4b1 100644 --- a/src/internal_events/lua.rs +++ b/src/internal_events/lua.rs @@ -1,8 +1,9 @@ -use metrics::{counter, gauge}; use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{counter, gauge}; use crate::transforms::lua::v2::BuildError; @@ -13,7 +14,7 @@ pub struct LuaGcTriggered { impl InternalEvent for LuaGcTriggered { fn emit(self) { - gauge!("lua_memory_used_bytes").set(self.used_memory as f64); + gauge!(GaugeName::LuaMemoryUsedBytes).set(self.used_memory as f64); } } @@ -32,7 +33,7 @@ impl InternalEvent for LuaScriptError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => mlua_error_code(&self.error), "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage::PROCESSING, @@ -62,7 +63,7 @@ impl InternalEvent for LuaBuildError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => lua_build_error_code(&self.error), "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage:: PROCESSING, diff --git a/src/internal_events/metric_to_log.rs b/src/internal_events/metric_to_log.rs index a536a1b24df24..a11ed286be17a 100644 --- a/src/internal_events/metric_to_log.rs +++ b/src/internal_events/metric_to_log.rs @@ -1,9 +1,8 @@ -use metrics::counter; use serde_json::Error; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct MetricToLogSerializeError { @@ -20,7 +19,7 @@ impl InternalEvent for MetricToLogSerializeError { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/mongodb_metrics.rs b/src/internal_events/mongodb_metrics.rs index 3cad1e4ef97f9..b0023085f025f 100644 --- a/src/internal_events/mongodb_metrics.rs +++ b/src/internal_events/mongodb_metrics.rs @@ -1,8 +1,7 @@ -use metrics::counter; use mongodb::{bson, error::Error as MongoError}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,12 +22,12 @@ impl InternalEvent for MongoDbMetricsEventsReceived<'_> { endpoint = self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -51,7 +50,7 @@ impl InternalEvent for MongoDbMetricsRequestError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -75,7 +74,7 @@ impl InternalEvent for MongoDbMetricsBsonParseError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, "endpoint" => self.endpoint.to_owned(), diff --git a/src/internal_events/mqtt.rs b/src/internal_events/mqtt.rs index 1e406a9494f1f..8d416c4c748ee 100644 --- a/src/internal_events/mqtt.rs +++ b/src/internal_events/mqtt.rs @@ -1,9 +1,10 @@ use std::fmt::Debug; -use metrics::counter; use rumqttc::ConnectionError; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct MqttConnectionError { @@ -20,7 +21,7 @@ impl InternalEvent for MqttConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "mqtt_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/nginx_metrics.rs b/src/internal_events/nginx_metrics.rs index a3384d4ef0faf..82cd7ea3878b7 100644 --- a/src/internal_events/nginx_metrics.rs +++ b/src/internal_events/nginx_metrics.rs @@ -1,7 +1,6 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,12 +22,12 @@ impl InternalEvent for NginxMetricsEventsReceived<'_> { endpoint = self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -51,7 +50,7 @@ impl InternalEvent for NginxMetricsRequestError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "endpoint" => self.endpoint.to_owned(), "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -76,7 +75,7 @@ impl InternalEvent for NginxMetricsStubStatusParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "endpoint" => self.endpoint.to_owned(), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/open.rs b/src/internal_events/open.rs index e48e0ac337820..58e9c9d40b3f8 100644 --- a/src/internal_events/open.rs +++ b/src/internal_events/open.rs @@ -6,9 +6,10 @@ use std::{ }, }; -use metrics::gauge; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, gauge, + internal_event::{GaugeName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct ConnectionOpen { @@ -17,7 +18,7 @@ pub struct ConnectionOpen { impl InternalEvent for ConnectionOpen { fn emit(self) { - gauge!("open_connections").set(self.count as f64); + gauge!(GaugeName::OpenConnections).set(self.count as f64); } } @@ -28,7 +29,7 @@ pub struct EndpointsActive { impl InternalEvent for EndpointsActive { fn emit(self) { - gauge!("active_endpoints").set(self.count as f64); + gauge!(GaugeName::ActiveEndpoints).set(self.count as f64); } } diff --git a/src/internal_events/parser.rs b/src/internal_events/parser.rs index 85a3e1383afa3..2610ae5458958 100644 --- a/src/internal_events/parser.rs +++ b/src/internal_events/parser.rs @@ -2,10 +2,11 @@ use std::borrow::Cow; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; fn truncate_string_at(s: &str, maxlen: usize) -> Cow<'_, str> { @@ -36,7 +37,7 @@ impl InternalEvent for ParserMatchError<'_> { field = &truncate_string_at(&String::from_utf8_lossy(self.value), 60)[..] ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "no_match_found", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -66,7 +67,7 @@ impl InternalEvent for ParserMissingFieldError<'_, DROP_ stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "field_not_found", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -97,7 +98,7 @@ impl InternalEvent for ParserConversionError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "type_conversion", "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/postgresql_metrics.rs b/src/internal_events/postgresql_metrics.rs index d6a58f968ab7f..62fa4d18a722d 100644 --- a/src/internal_events/postgresql_metrics.rs +++ b/src/internal_events/postgresql_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct PostgresqlMetricsCollectError<'a> { @@ -18,7 +19,7 @@ impl InternalEvent for PostgresqlMetricsCollectError<'_> { endpoint = %self.endpoint, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/process.rs b/src/internal_events/process.rs index 987724c2da18d..19701c6feaca8 100644 --- a/src/internal_events/process.rs +++ b/src/internal_events/process.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::{built_info, config}; @@ -17,7 +18,7 @@ impl InternalEvent for VectorStarted { arch = built_info::TARGET_ARCH, revision = built_info::VECTOR_BUILD_DESC.unwrap_or(""), ); - counter!("started_total").increment(1); + counter!(CounterName::StartedTotal).increment(1); } } @@ -34,7 +35,7 @@ impl InternalEvent for VectorReloaded<'_> { path = ?self.config_paths, internal_log_rate_limit = false, ); - counter!("reloaded_total").increment(1); + counter!(CounterName::ReloadedTotal).increment(1); } } @@ -59,7 +60,7 @@ impl InternalEvent for VectorStopped { target: "vector", message = "Vector has stopped.", ); - counter!("stopped_total").increment(1); + counter!(CounterName::StoppedTotal).increment(1); } } @@ -72,7 +73,7 @@ impl InternalEvent for VectorQuit { target: "vector", message = "Vector has quit.", ); - counter!("quit_total").increment(1); + counter!(CounterName::QuitTotal).increment(1); } } @@ -92,7 +93,7 @@ impl InternalEvent for VectorReloadError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reload", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, @@ -115,7 +116,7 @@ impl InternalEvent for VectorConfigLoadError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "config_load", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, @@ -137,7 +138,7 @@ impl InternalEvent for VectorRecoveryError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "recovery", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/prometheus.rs b/src/internal_events/prometheus.rs index 27b28fb5ffafa..3ddd3c555f728 100644 --- a/src/internal_events/prometheus.rs +++ b/src/internal_events/prometheus.rs @@ -3,13 +3,14 @@ #[cfg(feature = "sources-prometheus-scrape")] use std::borrow::Cow; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, -}; #[cfg(feature = "sources-prometheus-scrape")] use vector_lib::prometheus::parser::ParserError; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, +}; #[cfg(feature = "sources-prometheus-scrape")] #[derive(Debug, NamedInternalEvent)] @@ -34,7 +35,7 @@ impl InternalEvent for PrometheusParseError<'_> { url = %self.url ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "url" => self.url.to_string(), @@ -57,7 +58,7 @@ impl InternalEvent for PrometheusRemoteWriteParseError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -77,7 +78,7 @@ impl InternalEvent for PrometheusNormalizationError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/pulsar.rs b/src/internal_events/pulsar.rs index a2212ef0d5971..0e2e11b061f3e 100644 --- a/src/internal_events/pulsar.rs +++ b/src/internal_events/pulsar.rs @@ -2,11 +2,10 @@ #[cfg(feature = "sources-pulsar")] use metrics::Counter; -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct PulsarSendingError { @@ -24,7 +23,7 @@ impl InternalEvent for PulsarSendingError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) @@ -51,7 +50,7 @@ impl InternalEvent for PulsarPropertyExtractionError { property_field = %self.property_field, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "extracting_property", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -77,21 +76,21 @@ pub struct PulsarErrorEventData { registered_event!( PulsarErrorEvent => { ack_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "acknowledge_message", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ), nack_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "negative_acknowledge_message", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ), read_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_message", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/redis.rs b/src/internal_events/redis.rs index 45d76f3670dcd..87b9bc6d7b2f8 100644 --- a/src/internal_events/redis.rs +++ b/src/internal_events/redis.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct RedisReceiveEventError { @@ -25,7 +26,7 @@ impl InternalEvent for RedisReceiveEventError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/reduce.rs b/src/internal_events/reduce.rs index 8917c5aad0a5a..0962c6332a439 100644 --- a/src/internal_events/reduce.rs +++ b/src/internal_events/reduce.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use vrl::{path::PathParseError, value::KeyString}; #[derive(Debug, NamedInternalEvent)] @@ -8,7 +9,7 @@ pub struct ReduceStaleEventFlushed; impl InternalEvent for ReduceStaleEventFlushed { fn emit(self) { - counter!("stale_events_flushed_total").increment(1); + counter!(CounterName::StaleEventsFlushedTotal).increment(1); } } @@ -28,7 +29,7 @@ impl InternalEvent for ReduceAddEventError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/remap.rs b/src/internal_events/remap.rs index 989a6aa064065..d0e6500d69341 100644 --- a/src/internal_events/remap.rs +++ b/src/internal_events/remap.rs @@ -1,8 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct RemapMappingError { @@ -21,7 +21,7 @@ impl InternalEvent for RemapMappingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/sample.rs b/src/internal_events/sample.rs index 3722f9530b668..825175dc42555 100644 --- a/src/internal_events/sample.rs +++ b/src/internal_events/sample.rs @@ -1,5 +1,7 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct SampleEventDiscarded; diff --git a/src/internal_events/sematext_metrics.rs b/src/internal_events/sematext_metrics.rs index 65d23c53e00ef..cb03742a5d3d1 100644 --- a/src/internal_events/sematext_metrics.rs +++ b/src/internal_events/sematext_metrics.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; use crate::event::metric::Metric; @@ -23,7 +24,7 @@ impl InternalEvent for SematextMetricsInvalidMetricError<'_> { kind = ?self.metric.kind(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_metric", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, @@ -49,7 +50,7 @@ impl InternalEvent for SematextMetricsEncodeEventError stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index f0b58ce33bbd7..d38788f46767c 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,10 +1,10 @@ use std::net::Ipv4Addr; -use metrics::{counter, histogram}; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, histogram, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -42,11 +42,11 @@ impl InternalEvent for SocketBytesReceived { %protocol, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => protocol, ) .increment(self.byte_size as u64); - histogram!("component_received_bytes").record(self.byte_size as f64); + histogram!(HistogramName::ComponentReceivedBytes).record(self.byte_size as f64); } } @@ -66,10 +66,12 @@ impl InternalEvent for SocketEventsReceived { byte_size = self.byte_size.get(), %mode, ); - counter!("component_received_events_total", "mode" => mode).increment(self.count as u64); - counter!("component_received_event_bytes_total", "mode" => mode) + counter!(CounterName::ComponentReceivedEventsTotal, "mode" => mode) + .increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal, "mode" => mode) .increment(self.byte_size.get() as u64); - histogram!("component_received_bytes", "mode" => mode).record(self.byte_size.get() as f64); + histogram!(HistogramName::ComponentReceivedBytes, "mode" => mode) + .record(self.byte_size.get() as f64); } } @@ -88,7 +90,7 @@ impl InternalEvent for SocketBytesSent { %protocol, ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => protocol, ) .increment(self.byte_size as u64); @@ -105,8 +107,9 @@ pub struct SocketEventsSent { impl InternalEvent for SocketEventsSent { fn emit(self) { trace!(message = "Events sent.", count = %self.count, byte_size = %self.byte_size.get()); - counter!("component_sent_events_total", "mode" => self.mode.as_str()).increment(self.count); - counter!("component_sent_event_bytes_total", "mode" => self.mode.as_str()) + counter!(CounterName::ComponentSentEventsTotal, "mode" => self.mode.as_str()) + .increment(self.count); + counter!(CounterName::ComponentSentEventBytesTotal, "mode" => self.mode.as_str()) .increment(self.byte_size.get() as u64); } } @@ -129,7 +132,7 @@ impl InternalEvent for SocketBindError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_bind", "error_type" => error_type::IO_FAILED, "stage" => error_stage::INITIALIZING, @@ -164,7 +167,7 @@ impl InternalEvent for SocketMulticastGroupJoinError { %interface, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_multicast_group_join", "error_type" => error_type::IO_FAILED, "stage" => error_stage::INITIALIZING, @@ -194,7 +197,7 @@ impl InternalEvent for SocketReceiveError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_receive", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -223,7 +226,7 @@ impl InternalEvent for SocketSendError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_send", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index 6dcd63084d8e6..bd3fde83c2318 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -7,11 +7,13 @@ pub use self::source::*; #[cfg(feature = "sinks-splunk_hec")] mod sink { - use metrics::{counter, gauge}; use serde_json::Error; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{ + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, + }, }; use crate::{ @@ -35,7 +37,7 @@ mod sink { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "serializing_json", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, @@ -63,13 +65,13 @@ mod sink { kind = ?self.kind, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::INVALID_METRIC, "stage" => error_stage::PROCESSING, ) .increment(1); counter!( - "component_discarded_events_total", + CounterName::ComponentDiscardedEventsTotal, "error_type" => error_type::INVALID_METRIC, "stage" => error_stage::PROCESSING, ) @@ -92,7 +94,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_response", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::SENDING, @@ -117,7 +119,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "indexer_ack_failed", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::SENDING, @@ -141,7 +143,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "indexer_ack_unavailable", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::SENDING, @@ -155,7 +157,7 @@ mod sink { impl InternalEvent for SplunkIndexerAcknowledgementAckAdded { fn emit(self) { - gauge!("splunk_pending_acks").increment(1.0); + gauge!(GaugeName::SplunkPendingAcks).increment(1.0); } } @@ -166,7 +168,7 @@ mod sink { impl InternalEvent for SplunkIndexerAcknowledgementAcksRemoved { fn emit(self) { - gauge!("splunk_pending_acks").decrement(self.count); + gauge!(GaugeName::SplunkPendingAcks).decrement(self.count); } } @@ -197,9 +199,10 @@ mod sink { #[cfg(feature = "sources-splunk_hec")] mod source { - use metrics::counter; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; + use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, + }; use crate::sources::splunk_hec::ApiError; @@ -218,7 +221,7 @@ mod source { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_request_body", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -243,7 +246,7 @@ mod source { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::AUTHENTICATION_FAILED, "stage" => error_stage::RECEIVING, ) @@ -257,7 +260,7 @@ mod source { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/statsd_sink.rs b/src/internal_events/statsd_sink.rs index 7f0ee81f48ddc..858084124b562 100644 --- a/src/internal_events/statsd_sink.rs +++ b/src/internal_events/statsd_sink.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; use crate::event::metric::{MetricKind, MetricValue}; @@ -24,7 +23,7 @@ impl InternalEvent for StatsdInvalidMetricError<'_> { kind = ?self.kind, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_metric", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/tag_cardinality_limit.rs b/src/internal_events/tag_cardinality_limit.rs index 83a82acd342ae..9906f5c320f6b 100644 --- a/src/internal_events/tag_cardinality_limit.rs +++ b/src/internal_events/tag_cardinality_limit.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent}, +}; #[derive(NamedInternalEvent)] pub struct TagCardinalityLimitRejectingEvent<'a> { @@ -20,13 +21,13 @@ impl InternalEvent for TagCardinalityLimitRejectingEvent<'_> { ); if self.include_extended_tags { counter!( - "tag_value_limit_exceeded_total", + CounterName::TagValueLimitExceededTotal, "metric_name" => self.metric_name.to_string(), "tag_key" => self.tag_key.to_string(), ) .increment(1); } else { - counter!("tag_value_limit_exceeded_total").increment(1); + counter!(CounterName::TagValueLimitExceededTotal).increment(1); } emit!(ComponentEventsDropped:: { @@ -54,13 +55,13 @@ impl InternalEvent for TagCardinalityLimitRejectingTag<'_> { ); if self.include_extended_tags { counter!( - "tag_value_limit_exceeded_total", + CounterName::TagValueLimitExceededTotal, "metric_name" => self.metric_name.to_string(), "tag_key" => self.tag_key.to_string(), ) .increment(1); } else { - counter!("tag_value_limit_exceeded_total").increment(1); + counter!(CounterName::TagValueLimitExceededTotal).increment(1); } } } @@ -76,6 +77,6 @@ impl InternalEvent for TagCardinalityValueLimitReached<'_> { message = "Value_limit reached for key. New values for this key will be rejected.", key = %self.key, ); - counter!("value_limit_reached_total").increment(1); + counter!(CounterName::ValueLimitReachedTotal).increment(1); } } diff --git a/src/internal_events/tcp.rs b/src/internal_events/tcp.rs index a121eef1e0327..1ec0cfb6322e5 100644 --- a/src/internal_events/tcp.rs +++ b/src/internal_events/tcp.rs @@ -1,8 +1,9 @@ use std::net::SocketAddr; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::{internal_events::SocketOutgoingConnectionError, tls::TlsError}; @@ -18,7 +19,7 @@ impl InternalEvent for TcpSocketConnectionEstablished { } else { debug!(message = "Connected.", peer_addr = "unknown"); } - counter!("connection_established_total", "mode" => "tcp").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "tcp").increment(1); } } @@ -41,7 +42,7 @@ pub struct TcpSocketConnectionShutdown; impl InternalEvent for TcpSocketConnectionShutdown { fn emit(self) { warn!(message = "Received EOF from the server, shutdown."); - counter!("connection_shutdown_total", "mode" => "tcp").increment(1); + counter!(CounterName::ConnectionShutdownTotal, "mode" => "tcp").increment(1); } } @@ -63,7 +64,7 @@ impl InternalEvent for TcpSocketError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::PROCESSING, ) @@ -100,7 +101,7 @@ impl InternalEvent for TcpSocketTlsConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "connection_failed", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -127,7 +128,7 @@ impl InternalEvent for TcpSendAckError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "ack_failed", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -152,7 +153,7 @@ impl InternalEvent for TcpBytesReceived { peer_addr = %self.peer_addr, ); counter!( - "component_received_bytes_total", "protocol" => "tcp" + CounterName::ComponentReceivedBytesTotal, "protocol" => "tcp" ) .increment(self.byte_size as u64); } diff --git a/src/internal_events/template.rs b/src/internal_events/template.rs index 147418042a0c6..db43081e2a22a 100644 --- a/src/internal_events/template.rs +++ b/src/internal_events/template.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; #[derive(NamedInternalEvent)] @@ -29,7 +30,7 @@ impl InternalEvent for TemplateRenderingError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::TEMPLATE_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/throttle.rs b/src/internal_events/throttle.rs index 274fedd7741b9..3757bbae40535 100644 --- a/src/internal_events/throttle.rs +++ b/src/internal_events/throttle.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub(crate) struct ThrottleEventDiscarded { @@ -23,7 +24,7 @@ impl InternalEvent for ThrottleEventDiscarded { // if we should change the specification wording? Sort of a similar situation to the // `error_code` tag for the component errors metric, where it's meant to be optional and // only specified when relevant. - counter!("events_discarded_total", "key" => self.key).increment(1); // Deprecated. + counter!(CounterName::EventsDiscardedTotal, "key" => self.key).increment(1); // Deprecated. } emit!(ComponentEventsDropped:: { diff --git a/src/internal_events/udp.rs b/src/internal_events/udp.rs index 0602d4fdb3329..31e5f5c6533a4 100644 --- a/src/internal_events/udp.rs +++ b/src/internal_events/udp.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; use crate::internal_events::SocketOutgoingConnectionError; @@ -14,7 +15,7 @@ pub struct UdpSocketConnectionEstablished; impl InternalEvent for UdpSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); - counter!("connection_established_total", "mode" => "udp").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "udp").increment(1); } } @@ -51,13 +52,13 @@ impl InternalEvent for UdpSendIncompleteError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) .increment(1); // deprecated - counter!("connection_send_errors_total", "mode" => "udp").increment(1); + counter!(CounterName::ConnectionSendErrorsTotal, "mode" => "udp").increment(1); emit!(ComponentEventsDropped:: { count: 1, reason }); } @@ -80,7 +81,7 @@ impl InternalEvent for UdpChunkingError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/src/internal_events/unix.rs b/src/internal_events/unix.rs index 6fab3348dc4d7..fc98ff7feacbd 100644 --- a/src/internal_events/unix.rs +++ b/src/internal_events/unix.rs @@ -2,11 +2,10 @@ use std::{io::Error, path::Path}; -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; use crate::internal_events::SocketOutgoingConnectionError; @@ -18,7 +17,7 @@ pub struct UnixSocketConnectionEstablished<'a> { impl InternalEvent for UnixSocketConnectionEstablished<'_> { fn emit(self) { debug!(message = "Connected.", path = ?self.path); - counter!("connection_established_total", "mode" => "unix").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "unix").increment(1); } } @@ -59,7 +58,7 @@ impl InternalEvent for UnixSocketError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::PROCESSING, ) @@ -84,7 +83,7 @@ impl InternalEvent for UnixSocketSendError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -112,7 +111,7 @@ impl InternalEvent for UnixSendIncompleteError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -139,7 +138,7 @@ impl InternalEvent for UnixSocketFileDeleteError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "delete_socket_file", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/websocket.rs b/src/internal_events/websocket.rs index 960aec1916133..b842407ee0313 100644 --- a/src/internal_events/websocket.rs +++ b/src/internal_events/websocket.rs @@ -5,14 +5,15 @@ use std::{ fmt::{Debug, Display, Formatter, Result}, }; -use metrics::{counter, histogram}; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use vector_common::{ internal_event::{error_stage, error_type}, json_size::JsonSize, }; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent}, +}; pub const PROTOCOL: &str = "websocket"; @@ -22,7 +23,7 @@ pub struct WebSocketConnectionEstablished; impl InternalEvent for WebSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); - counter!("connection_established_total").increment(1); + counter!(CounterName::ConnectionEstablishedTotal).increment(1); } } @@ -41,7 +42,7 @@ impl InternalEvent for WebSocketConnectionFailedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_connection_failed", "error_type" => error_type::CONNECTION_FAILED, @@ -57,7 +58,7 @@ pub struct WebSocketConnectionShutdown; impl InternalEvent for WebSocketConnectionShutdown { fn emit(self) { warn!(message = "Closed by the server."); - counter!("connection_shutdown_total").increment(1); + counter!(CounterName::ConnectionShutdownTotal).increment(1); } } @@ -76,7 +77,7 @@ impl InternalEvent for WebSocketConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_connection_error", "error_type" => error_type::WRITER_FAILED, @@ -121,7 +122,7 @@ impl InternalEvent for WebSocketBytesReceived<'_> { kind = %self.kind ); let counter = counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "url" => self.url.to_string(), "protocol" => self.protocol, "kind" => self.kind.to_string() @@ -150,17 +151,17 @@ impl InternalEvent for WebSocketMessageReceived<'_> { kind = %self.kind ); - let histogram = histogram!("component_received_events_count"); + let histogram = histogram!(HistogramName::ComponentReceivedEventsCount); histogram.record(self.count as f64); let counter = counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "uri" => self.url.to_string(), "protocol" => PROTOCOL, "kind" => self.kind.to_string() ); counter.increment(self.count as u64); let counter = counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "url" => self.url.to_string(), "protocol" => PROTOCOL, "kind" => self.kind.to_string() @@ -184,7 +185,7 @@ impl InternalEvent for WebSocketReceiveError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_receive_error", "error_type" => error_type::CONNECTION_FAILED, @@ -209,7 +210,7 @@ impl InternalEvent for WebSocketSendError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_send_error", "error_type" => error_type::CONNECTION_FAILED, diff --git a/src/internal_events/websocket_server.rs b/src/internal_events/websocket_server.rs index fd064c41f2f8a..bdace6b079da5 100644 --- a/src/internal_events/websocket_server.rs +++ b/src/internal_events/websocket_server.rs @@ -1,8 +1,9 @@ use std::{error::Error, fmt::Debug}; -use metrics::{counter, gauge}; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct WebSocketListenerConnectionEstablished { @@ -18,8 +19,8 @@ impl InternalEvent for WebSocketListenerConnectionEstablished { self.client_count ) ); - counter!("connection_established_total", &self.extra_tags).increment(1); - gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + counter!(CounterName::ConnectionEstablishedTotal, &self.extra_tags).increment(1); + gauge!(GaugeName::ActiveClients, &self.extra_tags).set(self.client_count as f64); } } @@ -49,7 +50,7 @@ impl InternalEvent for WebSocketListenerConnectionFailedError { ]); // Tags required by `component_errors_total` are dynamically added above. // ## skip check-validity-events ## - counter!("component_errors_total", &all_tags).increment(1); + counter!(CounterName::ComponentErrorsTotal, &all_tags).increment(1); } } @@ -67,8 +68,8 @@ impl InternalEvent for WebSocketListenerConnectionShutdown { self.client_count ) ); - counter!("connection_shutdown_total", &self.extra_tags).increment(1); - gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + counter!(CounterName::ConnectionShutdownTotal, &self.extra_tags).increment(1); + gauge!(GaugeName::ActiveClients, &self.extra_tags).set(self.client_count as f64); } } @@ -87,7 +88,7 @@ impl InternalEvent for WebSocketListenerSendError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "ws_server_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -104,8 +105,8 @@ pub struct WebSocketListenerMessageSent { impl InternalEvent for WebSocketListenerMessageSent { fn emit(self) { - counter!("websocket_messages_sent_total", &self.extra_tags).increment(1); - counter!("websocket_bytes_sent_total", &self.extra_tags) + counter!(CounterName::WebsocketMessagesSentTotal, &self.extra_tags).increment(1); + counter!(CounterName::WebsocketBytesSentTotal, &self.extra_tags) .increment(self.message_size as u64); } } diff --git a/src/internal_events/windows.rs b/src/internal_events/windows.rs index 7b36e2fa62466..5ea6bae62aa25 100644 --- a/src/internal_events/windows.rs +++ b/src/internal_events/windows.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct WindowsServiceStart<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for WindowsServiceStart<'_> { "Started Windows Service.", ); counter!( - "windows_service_start_total", + CounterName::WindowsServiceStartTotal, "already_started" => self.already_started.to_string(), ) .increment(1); @@ -37,7 +38,7 @@ impl InternalEvent for WindowsServiceStop<'_> { "Stopped Windows Service.", ); counter!( - "windows_service_stop_total", + CounterName::WindowsServiceStopTotal, "already_stopped" => self.already_stopped.to_string(), ) .increment(1); @@ -55,7 +56,7 @@ impl InternalEvent for WindowsServiceRestart<'_> { name = ?self.name, "Restarted Windows Service." ); - counter!("windows_service_restart_total").increment(1) + counter!(CounterName::WindowsServiceRestartTotal).increment(1) } } @@ -70,7 +71,7 @@ impl InternalEvent for WindowsServiceInstall<'_> { name = ?self.name, "Installed Windows Service.", ); - counter!("windows_service_install_total").increment(1); + counter!(CounterName::WindowsServiceInstallTotal).increment(1); } } @@ -85,7 +86,7 @@ impl InternalEvent for WindowsServiceUninstall<'_> { name = ?self.name, "Uninstalled Windows Service.", ); - counter!("windows_service_uninstall_total").increment(1); + counter!(CounterName::WindowsServiceUninstallTotal).increment(1); } } @@ -104,7 +105,7 @@ impl InternalEvent for WindowsServiceDoesNotExistError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "service_missing", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/windows_event_log.rs b/src/internal_events/windows_event_log.rs index ae5363e2c52be..7ec356ba253ff 100644 --- a/src/internal_events/windows_event_log.rs +++ b/src/internal_events/windows_event_log.rs @@ -1,8 +1,7 @@ -use metrics::counter; use tracing::error; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[derive(Debug, NamedInternalEvent)] @@ -25,7 +24,7 @@ impl InternalEvent for WindowsEventLogParseError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parse_failed", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -54,7 +53,7 @@ impl InternalEvent for WindowsEventLogQueryError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "query_failed", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -81,7 +80,7 @@ impl InternalEvent for WindowsEventLogBookmarkError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "bookmark_failed", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_telemetry/allocations/mod.rs b/src/internal_telemetry/allocations/mod.rs index a2680256ec167..15ad82b3665fa 100644 --- a/src/internal_telemetry/allocations/mod.rs +++ b/src/internal_telemetry/allocations/mod.rs @@ -11,8 +11,11 @@ use std::{ }; use arr_macro::arr; -use metrics::{counter, gauge}; use rand_distr::num_traits::ToPrimitive; +use vector_common::{ + counter, gauge, + internal_event::{CounterName, GaugeName}, +}; use self::allocator::Tracer; pub(crate) use self::allocator::{ @@ -144,26 +147,26 @@ pub fn init_allocation_tracing() { let group_info = group.lock().unwrap(); if allocations_diff > 0 { counter!( - "component_allocated_bytes_total", "component_kind" => group_info.component_kind.clone(), + CounterName::ComponentAllocatedBytesTotal, "component_kind" => group_info.component_kind.clone(), "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone()).increment(allocations_diff); } if deallocations_diff > 0 { counter!( - "component_deallocated_bytes_total", "component_kind" => group_info.component_kind.clone(), + CounterName::ComponentDeallocatedBytesTotal, "component_kind" => group_info.component_kind.clone(), "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone()).increment(deallocations_diff); } if mem_used_diff > 0 { gauge!( - "component_allocated_bytes", "component_type" => group_info.component_type.clone(), + GaugeName::ComponentAllocatedBytes, "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone(), "component_kind" => group_info.component_kind.clone()) .increment(mem_used_diff.to_f64().expect("failed to convert mem_used from int to float")); } if mem_used_diff < 0 { gauge!( - "component_allocated_bytes", "component_type" => group_info.component_type.clone(), + GaugeName::ComponentAllocatedBytes, "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone(), "component_kind" => group_info.component_kind.clone()) .decrement(-mem_used_diff.to_f64().expect("failed to convert mem_used from int to float")); diff --git a/src/sources/internal_metrics.rs b/src/sources/internal_metrics.rs index f576efa2a71a1..76dfab8849997 100644 --- a/src/sources/internal_metrics.rs +++ b/src/sources/internal_metrics.rs @@ -196,8 +196,13 @@ impl InternalMetrics<'_> { mod tests { use std::collections::BTreeMap; - use metrics::{counter, gauge, histogram}; - use vector_lib::{metric_tags, metrics::Controller}; + use strum::IntoEnumIterator; + use vector_lib::{ + counter, gauge, histogram, + internal_event::{CounterName, GaugeName, HistogramName}, + metric_tags, + metrics::Controller, + }; use super::*; use crate::{ @@ -223,14 +228,20 @@ mod tests { // There *seems* to be a race condition here (CI was flaky), so add a slight delay. std::thread::sleep(std::time::Duration::from_millis(300)); - gauge!("foo").set(1.0); - gauge!("foo").set(2.0); - counter!("bar").increment(3); - counter!("bar").increment(4); - histogram!("baz").record(5.0); - histogram!("baz").record(6.0); - histogram!("quux", "host" => "foo").record(8.0); - histogram!("quux", "host" => "foo").record(8.1); + let gauge_name = GaugeName::iter().next().unwrap(); + let counter_name = CounterName::iter().next().unwrap(); + let mut histogram_iter = HistogramName::iter(); + let histogram_name = histogram_iter.next().unwrap(); + let histogram_tagged_name = histogram_iter.next().unwrap(); + + gauge!(gauge_name).set(1.0); + gauge!(gauge_name).set(2.0); + counter!(counter_name).increment(3); + counter!(counter_name).increment(4); + histogram!(histogram_name).record(5.0); + histogram!(histogram_name).record(6.0); + histogram!(histogram_tagged_name, "host" => "foo").record(8.0); + histogram!(histogram_tagged_name, "host" => "foo").record(8.1); let controller = Controller::get().expect("no controller"); @@ -243,10 +254,16 @@ mod tests { .map(|metric| (metric.name().to_string(), metric)) .collect::>(); - assert_eq!(&MetricValue::Gauge { value: 2.0 }, output["foo"].value()); - assert_eq!(&MetricValue::Counter { value: 7.0 }, output["bar"].value()); + assert_eq!( + &MetricValue::Gauge { value: 2.0 }, + output[gauge_name.as_str()].value() + ); + assert_eq!( + &MetricValue::Counter { value: 7.0 }, + output[counter_name.as_str()].value() + ); - match &output["baz"].value() { + match &output[histogram_name.as_str()].value() { MetricValue::AggregatedHistogram { buckets, count, @@ -263,7 +280,7 @@ mod tests { _ => panic!("wrong type"), } - match &output["quux"].value() { + match &output[histogram_tagged_name.as_str()].value() { MetricValue::AggregatedHistogram { buckets, count, @@ -282,7 +299,7 @@ mod tests { } let labels = metric_tags!("host" => "foo"); - assert_eq!(Some(&labels), output["quux"].tags()); + assert_eq!(Some(&labels), output[histogram_tagged_name.as_str()].tags()); } async fn event_from_config(config: InternalMetricsConfig) -> Event { diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 8948109fa7fb7..a25a99d337a26 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -8,7 +8,6 @@ use std::{ use futures::{FutureExt, StreamExt, TryStreamExt, stream::FuturesOrdered}; use futures_util::stream::FuturesUnordered; -use metrics::gauge; use stream_cancel::{StreamExt as StreamCancelExt, Trigger, Tripwire}; use tokio::{ select, @@ -31,6 +30,7 @@ use vector_lib::{ source_sender::{CHUNK_SIZE, SourceSenderItem}, transform::update_runtime_schema_definition, }; +use vector_lib::{gauge, internal_event::GaugeName}; use vector_vrl_metrics::MetricsStorage; use super::{ @@ -627,7 +627,7 @@ impl<'a> Builder<'a> { let utilization_sender = self .utilization_registry - .add_component(key.clone(), gauge!("utilization")); + .add_component(key.clone(), gauge!(GaugeName::Utilization)); let component_key = key.clone(); let sink = async move { debug!("Sink starting."); @@ -747,7 +747,7 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry - .add_component(node.key.clone(), gauge!("utilization")); + .add_component(node.key.clone(), gauge!(GaugeName::Utilization)); let runner = Runner::new( t, input_rx, @@ -803,7 +803,7 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry - .add_component(key.clone(), gauge!("utilization")); + .add_component(key.clone(), gauge!(GaugeName::Utilization)); let output_sender = sender.clone(); let input_rx = Utilization::new(sender, key.clone(), input_rx.into_stream()); diff --git a/src/utilization.rs b/src/utilization.rs index b3496b998f1c5..21d5f670bf1d9 100644 --- a/src/utilization.rs +++ b/src/utilization.rs @@ -344,6 +344,9 @@ impl OutputUtilization { mod tests { use mock_instant::global::MockClock; use serial_test::serial; + use strum::IntoEnumIterator; + use vector_lib::gauge; + use vector_lib::internal_event::GaugeName; use super::*; @@ -353,7 +356,7 @@ mod tests { MockClock::set_time(Duration::from_secs(100)); Timer::new( - metrics::gauge!("test_utilization"), + gauge!(GaugeName::iter().next().unwrap()), #[cfg(debug_assertions)] "test_component".into(), ) @@ -522,7 +525,7 @@ mod tests { let (mut emitter, registry) = UtilizationEmitter::new(); let key = ComponentKey::from("test_transform"); - let sender = registry.add_component(key.clone(), metrics::gauge!("utilization")); + let sender = registry.add_component(key.clone(), gauge!(GaugeName::Utilization)); let output_sender = sender.clone(); // Upstream channel carrying EventArrays. From 48be543ff3b84dfbd56a49cc6a0a0aac450bbceb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 May 2026 10:01:46 -0400 Subject: [PATCH 142/364] fix(aws_sqs source): scope HistogramName import to s3 module (#25353) `HistogramName` was only used inside the `mod s3` block (gated on `sources-aws_s3`) but was imported at the parent module level under a broader cfg, breaking `cargo check --features sources-aws_sqs` with `#[deny(warnings)]`. Co-authored-by: Claude Opus 4.7 (1M context) --- src/internal_events/aws_sqs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 51e8b78d23941..c04cb21b117fb 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -6,7 +6,7 @@ use vector_lib::counter; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs"))] use vector_lib::{ NamedInternalEvent, - internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[cfg(feature = "sources-aws_s3")] @@ -18,6 +18,7 @@ mod s3 { SendMessageBatchRequestEntry, SendMessageBatchResultEntry, }; use vector_lib::histogram; + use vector_lib::internal_event::HistogramName; use super::*; use crate::sources::aws_s3::sqs::ProcessingError; From 96db40ef65b0d1246f06581987e5b6428468edf0 Mon Sep 17 00:00:00 2001 From: ArunPiduguDD Date: Mon, 4 May 2026 13:47:01 -0400 Subject: [PATCH 143/364] enhancement(metrics): Make transform-related functions in aggregate & tag cardinality transforms public (#25358) Make transform-related functions in aggregate & tag cardinality processors public --- src/transforms/aggregate.rs | 4 ++-- src/transforms/tag_cardinality_limit/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transforms/aggregate.rs b/src/transforms/aggregate.rs index c00315ad4871a..67de0815c6d29 100644 --- a/src/transforms/aggregate.rs +++ b/src/transforms/aggregate.rs @@ -181,7 +181,7 @@ impl Aggregate { }) } - fn record(&mut self, event: Event) { + pub fn record(&mut self, event: Event) { let (series, data, metadata) = event.into_metric().into_parts(); match &mut self.mode { @@ -300,7 +300,7 @@ impl Aggregate { } } - fn flush_into(&mut self, output: &mut Vec) { + pub fn flush_into(&mut self, output: &mut Vec) { let map = std::mem::take(&mut self.map); for (series, entry) in map.clone().into_iter() { let mut metric = Metric::from_parts(series, entry.0, entry.1); diff --git a/src/transforms/tag_cardinality_limit/mod.rs b/src/transforms/tag_cardinality_limit/mod.rs index 2f26e214a5b61..6b936c59ce02c 100644 --- a/src/transforms/tag_cardinality_limit/mod.rs +++ b/src/transforms/tag_cardinality_limit/mod.rs @@ -121,7 +121,7 @@ impl TagCardinalityLimit { .insert(value.clone()); } - fn transform_one(&mut self, mut event: Event) -> Option { + pub fn transform_one(&mut self, mut event: Event) -> Option { let metric = event.as_mut_metric(); let metric_name = metric.name().to_string(); let metric_namespace = metric.namespace().map(|n| n.to_string()); From 8073e93b48352f781aace209f816ac55280a8935 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 May 2026 15:13:10 -0400 Subject: [PATCH 144/364] chore(ci): rename WIP label workflows to docs review label (#25355) Rename `add_wip_label.yml` and `remove_wip_label.yml` to `add_docs_review_label.yml` and `remove_docs_review_label.yml`, and switch the managed label from "work in progress" to "docs review on hold". The label string is now hoisted to a workflow-level `LABEL_NAME` env var so future renames are a one-line edit. Header comments note we can promote these to a reusable `workflow_call` workflow if a second use case appears. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/CODEOWNERS | 2 +- ...ip_label.yml => add_docs_review_label.yml} | 12 +++++++++--- ...label.yml => remove_docs_review_label.yml} | 19 +++++++++++++------ 3 files changed, 23 insertions(+), 10 deletions(-) rename .github/workflows/{add_wip_label.yml => add_docs_review_label.yml} (81%) rename .github/workflows/{remove_wip_label.yml => remove_docs_review_label.yml} (68%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ecd6fb374af9c..7534be2fb8652 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance -# Keep documentation team paths in sync with .github/workflows/add_wip_label.yml +# Keep documentation team paths in sync with .github/workflows/add_docs_review_label.yml docs/ @vectordotdev/vector @vectordotdev/documentation website/ @vectordotdev/vector website/content @vectordotdev/vector @vectordotdev/documentation diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_docs_review_label.yml similarity index 81% rename from .github/workflows/add_wip_label.yml rename to .github/workflows/add_docs_review_label.yml index 7924d4fade926..23f9a697397af 100644 --- a/.github/workflows/add_wip_label.yml +++ b/.github/workflows/add_docs_review_label.yml @@ -1,4 +1,7 @@ -name: Add Work In Progress Label +# Adds a label to PRs that touch documentation paths until a maintainer approves. +# Currently dedicated to the docs review flow. If a second use case appears, promote +# this into a reusable workflow (`workflow_call`) with the label as an input. +name: Add Docs Review Label permissions: pull-requests: write @@ -12,8 +15,11 @@ on: - "website/content/**" - "website/cue/reference/**" +env: + LABEL_NAME: "docs review on hold" + jobs: - add_wip_label: + add_label: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -47,4 +53,4 @@ jobs: - if: steps.check.outputs.skip != 'true' uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 with: - labels: "work in progress" + labels: ${{ env.LABEL_NAME }} diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_docs_review_label.yml similarity index 68% rename from .github/workflows/remove_wip_label.yml rename to .github/workflows/remove_docs_review_label.yml index b4750b0f2b0d6..093c5a4bd5e90 100644 --- a/.github/workflows/remove_wip_label.yml +++ b/.github/workflows/remove_docs_review_label.yml @@ -1,4 +1,7 @@ -name: Remove Work In Progress Label +# Removes the docs review label once a maintainer approves the PR. +# Currently dedicated to the docs review flow. If a second use case appears, promote +# this into a reusable workflow (`workflow_call`) with the label as an input. +name: Remove Docs Review Label permissions: issues: write @@ -8,8 +11,11 @@ on: pull_request_review: types: [submitted] +env: + LABEL_NAME: "docs review on hold" + jobs: - remove_wip_label: + remove_label: if: github.event.review.state == 'approved' runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -27,20 +33,21 @@ jobs: return; } + const labelName = process.env.LABEL_NAME; const { data: labels } = await github.rest.issues.listLabelsOnIssue({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.pull_request.number, }); - const hasLabel = labels.some(l => l.name === 'work in progress'); + const hasLabel = labels.some(l => l.name === labelName); if (!hasLabel) { - core.info('Label "work in progress" not found, skipping'); + core.info(`Label "${labelName}" not found, skipping`); return; } await github.rest.issues.removeLabel({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.pull_request.number, - name: 'work in progress', + name: labelName, }); - core.info('Removed "work in progress" label'); + core.info(`Removed "${labelName}" label`); From 4524b52c921447dfca228338add190f0363b582e Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 4 May 2026 19:08:38 -0400 Subject: [PATCH 145/364] chore(deps): upgrade hickory-proto to 0.26.1, ignore RUSTSEC-2026-0119 (#25354) * chore(deps): upgrade hickory-proto to 0.26.1, ignore RUSTSEC-2026-0119 * Format * fix(dnstap source): include EDNS OPT record in additional_count --- Cargo.lock | 42 +- Cargo.toml | 2 +- deny.toml | 1 + lib/dnsmsg-parser/src/dns_message_parser.rs | 410 ++++++++++---------- lib/dnsmsg-parser/src/ede.rs | 11 +- 5 files changed, 226 insertions(+), 240 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e217d2d24c4f..cb014b2cd372a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3782,7 +3782,7 @@ version = "0.1.0" dependencies = [ "criterion", "data-encoding", - "hickory-proto 0.25.2", + "hickory-proto 0.26.1", "snafu 0.9.0", ] @@ -3796,7 +3796,7 @@ dependencies = [ "chrono", "chrono-tz", "dnsmsg-parser", - "hickory-proto 0.25.2", + "hickory-proto 0.26.1", "pastey", "prost 0.12.6", "prost-build 0.12.6", @@ -5134,7 +5134,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "hickory-proto 0.26.0", + "hickory-proto 0.26.1", "idna", "ipnet", "jni 0.22.4", @@ -5172,37 +5172,11 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.25.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" dependencies = [ - "async-trait", "bitflags 2.10.0", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna", - "ipnet", - "once_cell", - "rand 0.9.4", - "ring", - "rustls-pki-types", - "thiserror 2.0.17", - "time", - "tinyvec", - "tracing 0.1.44", - "url", -] - -[[package]] -name = "hickory-proto" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a916d0494600d99ecb15aadfab677ad97c4de559e8f1af0c129353a733ac1fcc" -dependencies = [ "data-encoding", "idna", "ipnet", @@ -5211,7 +5185,9 @@ dependencies = [ "prefix-trie", "rand 0.10.1", "ring", + "rustls-pki-types", "thiserror 2.0.17", + "time", "tinyvec", "tracing 0.1.44", "url", @@ -5247,7 +5223,7 @@ dependencies = [ "cfg-if", "futures-util", "hickory-net", - "hickory-proto 0.26.0", + "hickory-proto 0.26.1", "ipconfig", "ipnet", "jni 0.22.4", @@ -12854,7 +12830,7 @@ dependencies = [ "headers", "heim", "hex", - "hickory-proto 0.25.2", + "hickory-proto 0.26.1", "hostname 0.4.2", "http 0.2.12", "http 1.3.1", diff --git a/Cargo.toml b/Cargo.toml index e9cf7da0f400b..f28e8feabbfd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,7 +160,7 @@ flate2 = { version = "1.1.2", default-features = false, features = ["zlib-rs"] } futures = { version = "0.3.31", default-features = false, features = ["std"] } futures-util = { version = "0.3.29", default-features = false } glob = { version = "0.3.3", default-features = false } -hickory-proto = { version = "0.25.2", default-features = false, features = ["dnssec-ring"] } +hickory-proto = { version = "0.26.1", default-features = false, features = ["dnssec-ring"] } humantime = { version = "2.3.0", default-features = false } indexmap = { version = "2.11.0", default-features = false, features = ["serde", "std"] } indoc = { version = "2.0.7" } diff --git a/deny.toml b/deny.toml index 0d563622a509e..37db54adf7956 100644 --- a/deny.toml +++ b/deny.toml @@ -50,4 +50,5 @@ ignore = [ { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 unsound with custom logger - transitive dependency, upstream crates have not updated to rand 0.9+" }, { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, { id = "RUSTSEC-2026-0105", reason = "core2 is unmaintained and all versions yanked - transitive dependency via libflate -> apache-avro, no safe upgrade available" }, + { id = "RUSTSEC-2026-0119", reason = "hickory-proto 0.24 - unpatched crate (https://github.com/mongodb/mongo-rust-driver/pull/1682)" }, ] diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index d0b271b2e7930..a22b630ffaca4 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -7,16 +7,13 @@ use hickory_proto::{ PublicKey, SupportedAlgorithms, Verifier, rdata::{CDNSKEY, CDS, DNSKEY, DNSSECRData, DS}, }, - op::{Query, message::Message as TrustDnsMessage}, + op::{Message as TrustDnsMessage, Query}, rr::{ - Name, RecordType, + Name, RData, Record, RecordType, rdata::{ A, AAAA, NULL, OPT, SVCB, - caa::Property, opt::{EdnsCode, EdnsOption}, }, - record_data::RData, - resource::Record, }, serialize::binary::{BinDecodable, BinDecoder}, }; @@ -101,8 +98,11 @@ impl DnsMessageParser { } pub fn parse_as_query_message(&mut self) -> DnsParserResult { - let msg = TrustDnsMessage::from_vec(&self.raw_message) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + let msg = TrustDnsMessage::from_vec(&self.raw_message).map_err(|source| { + DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + } + })?; let header = parse_dns_query_message_header(&msg); let edns_section = parse_edns(&msg).transpose()?; let rcode_high = edns_section.as_ref().map_or(0, |edns| edns.extended_rcode); @@ -113,16 +113,19 @@ impl DnsMessageParser { response: parse_response_code(response_code), header, question_section: self.parse_dns_query_message_question_section(&msg), - answer_section: self.parse_dns_message_section(msg.answers())?, - authority_section: self.parse_dns_message_section(msg.name_servers())?, - additional_section: self.parse_dns_message_section(msg.additionals())?, + answer_section: self.parse_dns_message_section(&msg.answers)?, + authority_section: self.parse_dns_message_section(&msg.authorities)?, + additional_section: self.parse_dns_message_section(&msg.additionals)?, opt_pseudo_section: edns_section, }) } pub fn parse_as_update_message(&mut self) -> DnsParserResult { - let msg = TrustDnsMessage::from_vec(&self.raw_message) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + let msg = TrustDnsMessage::from_vec(&self.raw_message).map_err(|source| { + DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + } + })?; let header = parse_dns_update_message_header(&msg); let response_code = (u16::from(header.rcode)) & 0x000F; Ok(DnsUpdateMessage { @@ -130,9 +133,9 @@ impl DnsMessageParser { response: parse_response_code(response_code), header, zone_to_update: self.parse_dns_update_message_zone_section(&msg)?, - prerequisite_section: self.parse_dns_message_section(msg.answers())?, - update_section: self.parse_dns_message_section(msg.name_servers())?, - additional_section: self.parse_dns_message_section(msg.additionals())?, + prerequisite_section: self.parse_dns_message_section(&msg.answers)?, + update_section: self.parse_dns_message_section(&msg.authorities)?, + additional_section: self.parse_dns_message_section(&msg.additionals)?, }) } @@ -141,7 +144,7 @@ impl DnsMessageParser { dns_message: &TrustDnsMessage, ) -> Vec { dns_message - .queries() + .queries .iter() .map(|query| self.parse_dns_query_question(query)) .collect() @@ -161,7 +164,7 @@ impl DnsMessageParser { dns_message: &TrustDnsMessage, ) -> DnsParserResult { let zones = dns_message - .queries() + .queries .iter() .map(|query| self.parse_dns_query_question(query).into()) .collect::>(); @@ -185,18 +188,18 @@ impl DnsMessageParser { } pub(crate) fn parse_dns_record(&mut self, record: &Record) -> DnsParserResult { - let record_data = match record.data() { - RData::Unknown { code, rdata } => self.format_unknown_rdata((*code).into(), rdata), + let record_data = match &record.data { + RData::Unknown { code, rdata } => self.format_unknown_rdata(u16::from(*code), rdata), RData::Update0(_) => Ok((Some(String::from("")), None)), // Previously none value rdata => self.format_rdata(rdata), }?; Ok(DnsRecord { - name: record.name().to_string_with_options(&self.options), - class: record.dns_class().to_string(), + name: record.name.to_string_with_options(&self.options), + class: record.dns_class.to_string(), record_type: format_record_type(record.record_type()), record_type_id: u16::from(record.record_type()), - ttl: record.ttl(), + ttl: record.ttl, rdata: record_data.0, rdata_bytes: record_data.1, }) @@ -381,30 +384,30 @@ impl DnsMessageParser { match code { dns_message::RTYPE_MB => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let madname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(madname), None)) } dns_message::RTYPE_MG => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let mgname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(mgname), None)) } dns_message::RTYPE_MR => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let newname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(newname), None)) } - dns_message::RTYPE_WKS => self.parse_wks_rdata(rdata.anything()), + dns_message::RTYPE_WKS => self.parse_wks_rdata(&rdata.anything), dns_message::RTYPE_MINFO => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let rmailbx = Self::parse_domain_name(&mut decoder, &options)?; let emailbx = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{rmailbx} {emailbx}")), None)) @@ -412,7 +415,7 @@ impl DnsMessageParser { dns_message::RTYPE_RP => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let mbox = Self::parse_domain_name(&mut decoder, &options)?; let txt = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{mbox} {txt}")), None)) @@ -420,14 +423,14 @@ impl DnsMessageParser { dns_message::RTYPE_AFSDB => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let subtype = parse_u16(&mut decoder)?; let hostname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{subtype} {hostname}")), None)) } dns_message::RTYPE_X25 => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let psdn_address = parse_character_string(&mut decoder)?; Ok(( Some(format!( @@ -439,7 +442,7 @@ impl DnsMessageParser { } dns_message::RTYPE_ISDN => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let address = parse_character_string(&mut decoder)?; if decoder.is_empty() { Ok(( @@ -464,14 +467,14 @@ impl DnsMessageParser { dns_message::RTYPE_RT => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let intermediate_host = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {intermediate_host}")), None)) } dns_message::RTYPE_NSAP => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let rdata_len = raw_rdata.len() as u16; let nsap_rdata = HEXUPPER.encode(&parse_vec_with_u16_len(&mut decoder, rdata_len)?); @@ -480,27 +483,27 @@ impl DnsMessageParser { dns_message::RTYPE_PX => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let map822 = Self::parse_domain_name(&mut decoder, &options)?; let mapx400 = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {map822} {mapx400}")), None)) } - dns_message::RTYPE_LOC => self.parse_loc_rdata(rdata.anything()), + dns_message::RTYPE_LOC => self.parse_loc_rdata(&rdata.anything), dns_message::RTYPE_KX => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let exchanger = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {exchanger}")), None)) } - dns_message::RTYPE_A6 => self.parse_a6_rdata(rdata.anything()), + dns_message::RTYPE_A6 => self.parse_a6_rdata(&rdata.anything), dns_message::RTYPE_SINK => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let meaning = parse_u8(&mut decoder)?; let coding = parse_u8(&mut decoder)?; @@ -511,10 +514,10 @@ impl DnsMessageParser { Ok((Some(format!("{meaning} {coding} {subcoding} {data}")), None)) } - dns_message::RTYPE_APL => self.parse_apl_rdata(rdata.anything()), + dns_message::RTYPE_APL => self.parse_apl_rdata(&rdata.anything), dns_message::RTYPE_DHCID => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let raw_data_len = raw_rdata.len() as u16; let digest = BASE64.encode(&parse_vec_with_u16_len(&mut decoder, raw_data_len)?); @@ -522,7 +525,7 @@ impl DnsMessageParser { } dns_message::RTYPE_SPF => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let mut text = String::new(); while !decoder.is_empty() { text.push('\"'); @@ -532,7 +535,7 @@ impl DnsMessageParser { Ok((Some(text.trim_end().to_string()), None)) } - _ => Ok((None, Some(rdata.anything().to_vec()))), + _ => Ok((None, Some(rdata.anything.clone()))), } } @@ -543,13 +546,13 @@ impl DnsMessageParser { RData::ANAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CNAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CERT(cert) => { - let crl = BASE64.encode(&cert.cert_data()); + let crl = BASE64.encode(&cert.cert_data); Ok(( Some(format!( "{} {} {} {}", - u16::from(cert.cert_type()), - cert.key_tag(), - cert.algorithm(), + u16::from(cert.cert_type), + cert.key_tag, + cert.algorithm, crl )), None, @@ -564,15 +567,15 @@ impl DnsMessageParser { RData::MX(mx) => { let srv_rdata = format!( "{} {}", - mx.preference(), - mx.exchange().to_string_with_options(&self.options), + mx.preference, + mx.exchange.to_string_with_options(&self.options), ); Ok((Some(srv_rdata), None)) } - RData::NULL(null) => Ok((Some(BASE64.encode(null.anything())), None)), + RData::NULL(null) => Ok((Some(BASE64.encode(&null.anything)), None)), RData::NS(ns) => Ok((Some(ns.to_string_with_options(&self.options)), None)), RData::OPENPGPKEY(key) => { - if let Ok(key_string) = String::from_utf8(Vec::from(key.public_key())) { + if let Ok(key_string) = String::from_utf8(key.public_key.clone()) { Ok((Some(format!("({})", &key_string)), None)) } else { Err(DnsMessageParserError::SimpleError { @@ -584,29 +587,29 @@ impl DnsMessageParser { RData::SOA(soa) => Ok(( Some(format!( "{} {} {} {} {} {} {}", - soa.mname().to_string_with_options(&self.options), - soa.rname().to_string_with_options(&self.options), - soa.serial(), - soa.refresh(), - soa.retry(), - soa.expire(), - soa.minimum() + soa.mname.to_string_with_options(&self.options), + soa.rname.to_string_with_options(&self.options), + soa.serial, + soa.refresh, + soa.retry, + soa.expire, + soa.minimum )), None, )), RData::SRV(srv) => { let srv_rdata = format!( "{} {} {} {}", - srv.priority(), - srv.weight(), - srv.port(), - srv.target().to_string_with_options(&self.options) + srv.priority, + srv.weight, + srv.port, + srv.target.to_string_with_options(&self.options) ); Ok((Some(srv_rdata), None)) } RData::TXT(txt) => { let txt_rdata = txt - .txt_data() + .txt_data .iter() .map(|value| { format!( @@ -623,41 +626,35 @@ impl DnsMessageParser { RData::CAA(caa) => { let caa_rdata = format!( "{} {} \"{}\"", - caa.issuer_critical() as u8, - caa.tag().as_str(), - match caa.tag() { - Property::Iodef => { - let url = caa.value_as_iodef().map_err(|source| { - DnsMessageParserError::TrustDnsError { source } - })?; - url.as_str().to_string() - } - Property::Issue | Property::IssueWild => { - let (option_name, vec_keyvalue) = - caa.value_as_issue().map_err(|source| { - DnsMessageParserError::TrustDnsError { source } - })?; - - let mut final_issuer = String::new(); - if let Some(name) = option_name { - final_issuer.push_str(&name.to_string_with_options(&self.options)); - for keyvalue in vec_keyvalue.iter() { - final_issuer.push_str("; "); - final_issuer.push_str(keyvalue.key()); - final_issuer.push('='); - final_issuer.push_str(keyvalue.value()); - } + caa.issuer_critical as u8, + &caa.tag, + if caa.tag.eq_ignore_ascii_case("iodef") { + let url = caa + .value_as_iodef() + .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + url.as_str().to_string() + } else if caa.tag.eq_ignore_ascii_case("issue") + || caa.tag.eq_ignore_ascii_case("issuewild") + { + let (option_name, vec_keyvalue) = caa + .value_as_issue() + .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + + let mut final_issuer = String::new(); + if let Some(name) = option_name { + final_issuer.push_str(&name.to_string_with_options(&self.options)); + for keyvalue in vec_keyvalue.iter() { + final_issuer.push_str("; "); + final_issuer.push_str(keyvalue.key()); + final_issuer.push('='); + final_issuer.push_str(keyvalue.value()); } - final_issuer.trim_end().to_string() - } - Property::Unknown(_) => { - let unknown = caa.raw_value(); - std::str::from_utf8(unknown) - .map_err(|source| DnsMessageParserError::Utf8ParsingError { - source, - })? - .to_string() } + final_issuer.trim_end().to_string() + } else { + std::str::from_utf8(&caa.value) + .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? + .to_string() } ); Ok((Some(caa_rdata), None)) @@ -666,52 +663,52 @@ impl DnsMessageParser { RData::TLSA(tlsa) => { let tlsa_rdata = format!( "{} {} {} {}", - u8::from(tlsa.cert_usage()), - u8::from(tlsa.selector()), - u8::from(tlsa.matching()), - HEXUPPER.encode(tlsa.cert_data()) + u8::from(tlsa.cert_usage), + u8::from(tlsa.selector), + u8::from(tlsa.matching), + HEXUPPER.encode(&tlsa.cert_data) ); Ok((Some(tlsa_rdata), None)) } RData::SSHFP(sshfp) => { let sshfp_rdata = format!( "{} {} {}", - Into::::into(sshfp.algorithm()), - Into::::into(sshfp.fingerprint_type()), - HEXUPPER.encode(sshfp.fingerprint()) + Into::::into(sshfp.algorithm), + Into::::into(sshfp.fingerprint_type), + HEXUPPER.encode(&sshfp.fingerprint) ); Ok((Some(sshfp_rdata), None)) } RData::NAPTR(naptr) => { let naptr_rdata = format!( r#"{} {} "{}" "{}" "{}" {}"#, - naptr.order(), - naptr.preference(), + naptr.order, + naptr.preference, escape_string_for_text_representation( - std::str::from_utf8(naptr.flags()) + std::str::from_utf8(&naptr.flags) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), escape_string_for_text_representation( - std::str::from_utf8(naptr.services()) + std::str::from_utf8(&naptr.services) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), escape_string_for_text_representation( - std::str::from_utf8(naptr.regexp()) + std::str::from_utf8(&naptr.regexp) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), - naptr.replacement().to_string_with_options(&self.options) + naptr.replacement.to_string_with_options(&self.options) ); Ok((Some(naptr_rdata), None)) } RData::HINFO(hinfo) => { let hinfo_data = format!( r#""{}" "{}""#, - std::str::from_utf8(hinfo.cpu()) + std::str::from_utf8(&hinfo.cpu) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })?, - std::str::from_utf8(hinfo.os()) + std::str::from_utf8(&hinfo.os) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })?, ); Ok((Some(hinfo_data), None)) @@ -794,37 +791,41 @@ impl DnsMessageParser { DNSSECRData::SIG(sig) => { let sig_rdata = format!( "{} {} {} {} {} {} {} {} {}", - match format_record_type(sig.type_covered()) { + match format_record_type(sig.input().type_covered) { Some(record_type) => record_type, None => String::from("Unknown record type"), }, - u8::from(sig.algorithm()), - sig.num_labels(), - sig.original_ttl(), - sig.sig_expiration().get(), // currently in epoch convert to human readable ? - sig.sig_inception().get(), // currently in epoch convert to human readable ? - sig.key_tag(), - sig.signer_name().to_string_with_options(&self.options), + u8::from(sig.input().algorithm), + sig.input().num_labels, + sig.input().original_ttl, + sig.input().sig_expiration.get(), // currently in epoch convert to human readable ? + sig.input().sig_inception.get(), // currently in epoch convert to human readable ? + sig.input().key_tag, + sig.input() + .signer_name + .to_string_with_options(&self.options), BASE64.encode(sig.sig()) ); Ok((Some(sig_rdata), None)) } - // RSIG is a derivation of SIG but choosing to keep this duplicate code in lieu of the alternative + // RRSIG is a derivation of SIG but choosing to keep this duplicate code in lieu of the alternative // which is to allocate to the heap with Box in order to deref. DNSSECRData::RRSIG(sig) => { let sig_rdata = format!( "{} {} {} {} {} {} {} {} {}", - match format_record_type(sig.type_covered()) { + match format_record_type(sig.input().type_covered) { Some(record_type) => record_type, None => String::from("Unknown record type"), }, - u8::from(sig.algorithm()), - sig.num_labels(), - sig.original_ttl(), - sig.sig_expiration().get(), // currently in epoch convert to human readable ? - sig.sig_inception().get(), // currently in epoch convert to human readable ? - sig.key_tag(), - sig.signer_name().to_string_with_options(&self.options), + u8::from(sig.input().algorithm), + sig.input().num_labels, + sig.input().original_ttl, + sig.input().sig_expiration.get(), // currently in epoch convert to human readable ? + sig.input().sig_inception.get(), // currently in epoch convert to human readable ? + sig.input().key_tag, + sig.input() + .signer_name + .to_string_with_options(&self.options), BASE64.encode(sig.sig()) ); Ok((Some(sig_rdata), None)) @@ -839,9 +840,7 @@ impl DnsMessageParser { ); Ok((Some(key_rdata), None)) } - DNSSECRData::Unknown { code: _, rdata } => { - Ok((None, Some(rdata.anything().to_vec()))) - } + DNSSECRData::Unknown { code: _, rdata } => Ok((None, Some(rdata.anything.clone()))), _ => Err(DnsMessageParserError::SimpleError { cause: format!("Unsupported rdata {rdata:?}"), }), @@ -870,9 +869,9 @@ fn format_record_type(record_type: RecordType) -> Option { fn format_svcb_record(svcb: &SVCB, options: &DnsParserOptions) -> String { format!( "{} {} {}", - svcb.svc_priority(), - svcb.target_name().to_string_with_options(options), - svcb.svc_params() + svcb.svc_priority, + svcb.target_name.to_string_with_options(options), + svcb.svc_params .iter() .map(|(key, value)| format!(r#"{}="{}""#, key, value.to_string().trim_end_matches(','))) .collect::>() @@ -965,38 +964,38 @@ fn parse_response_code(rcode: u16) -> Option<&'static str> { fn parse_dns_query_message_header(dns_message: &TrustDnsMessage) -> QueryHeader { QueryHeader { - id: dns_message.header().id(), - opcode: dns_message.header().op_code().into(), - rcode: dns_message.header().response_code(), - qr: dns_message.header().message_type() as u8, - aa: dns_message.header().authoritative(), - tc: dns_message.header().truncated(), - rd: dns_message.header().recursion_desired(), - ra: dns_message.header().recursion_available(), - ad: dns_message.header().authentic_data(), - cd: dns_message.header().checking_disabled(), - question_count: dns_message.header().query_count(), - answer_count: dns_message.header().answer_count(), - authority_count: dns_message.header().name_server_count(), - additional_count: dns_message.header().additional_count(), + id: dns_message.id, + opcode: dns_message.op_code.into(), + rcode: dns_message.response_code, + qr: dns_message.message_type as u8, + aa: dns_message.authoritative, + tc: dns_message.truncation, + rd: dns_message.recursion_desired, + ra: dns_message.recursion_available, + ad: dns_message.authentic_data, + cd: dns_message.checking_disabled, + question_count: dns_message.queries.len() as u16, + answer_count: dns_message.answers.len() as u16, + authority_count: dns_message.authorities.len() as u16, + additional_count: dns_message.additionals.len() as u16 + dns_message.edns.is_some() as u16, } } fn parse_dns_update_message_header(dns_message: &TrustDnsMessage) -> UpdateHeader { UpdateHeader { - id: dns_message.header().id(), - opcode: dns_message.header().op_code().into(), - rcode: dns_message.header().response_code(), - qr: dns_message.header().message_type() as u8, - zone_count: dns_message.header().query_count(), - prerequisite_count: dns_message.header().answer_count(), - update_count: dns_message.header().name_server_count(), - additional_count: dns_message.header().additional_count(), + id: dns_message.id, + opcode: dns_message.op_code.into(), + rcode: dns_message.response_code, + qr: dns_message.message_type as u8, + zone_count: dns_message.queries.len() as u16, + prerequisite_count: dns_message.answers.len() as u16, + update_count: dns_message.authorities.len() as u16, + additional_count: dns_message.additionals.len() as u16 + dns_message.edns.is_some() as u16, } } fn parse_edns(dns_message: &TrustDnsMessage) -> Option> { - dns_message.extensions().as_ref().map(|edns| { + dns_message.edns.as_ref().map(|edns| { parse_edns_options(edns.options()).map(|(ede, rest)| OptPseudoSection { extended_rcode: edns.rcode_high(), version: edns.version(), @@ -1014,10 +1013,11 @@ fn parse_edns_options(edns: &OPT) -> DnsParserResult<(Vec, Vec) -> DnsParserResult { Ok(::read(decoder) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })? + .map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + })? .to_string()) } fn parse_ipv4_address(decoder: &mut BinDecoder<'_>) -> DnsParserResult { Ok(::read(decoder) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })? + .map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + })? .to_string()) } fn parse_domain_name(decoder: &mut BinDecoder<'_>) -> DnsParserResult { - Name::read(decoder).map_err(|source| DnsMessageParserError::TrustDnsError { source }) + Name::read(decoder).map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + }) } fn escape_string_for_text_representation(original_string: String) -> String { @@ -1305,11 +1311,13 @@ mod tests { dnssec::{ Algorithm as DNSSEC_Algorithm, DigestType, Nsec3HashAlgorithm, PublicKeyBuf, rdata::{ - KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, SIG, + KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, key::{KeyTrust, KeyUsage, Protocol}, + sig::SigInput, }, }, rr::{ + SerialNumber, domain::Name, rdata::{ CAA, CERT, CSYNC, HINFO, HTTPS, NAPTR, OPT, SSHFP, TLSA, TXT, @@ -1435,7 +1443,10 @@ mod tests { .expect_err("Expected TrustDnsError."); match err { DnsMessageParserError::TrustDnsError { source: e } => { - assert_eq!(e.to_string(), "unexpected end of input reached") + assert_eq!( + e.to_string(), + "decoding error: unexpected end of input reached" + ) } DnsMessageParserError::SimpleError { cause: e } => { panic!("Expected TrustDnsError, got {}.", &e) @@ -1859,29 +1870,26 @@ mod tests { #[test] fn test_format_rdata_for_sig_type() { - let rdata = RData::DNSSEC(DNSSECRData::SIG(SIG::new( - RecordType::NULL, - DNSSEC_Algorithm::RSASHA256, + // SIG wire: type_covered=NULL(10), alg=8, labels=0, orig_ttl=0, expire=2, inception=1, + // keytag=5, signer=www.example.com, sig=[0..=31] + let mut wire: Vec = Vec::new(); + wire.extend_from_slice(&10u16.to_be_bytes()); + wire.push(8u8); + wire.push(0u8); + wire.extend_from_slice(&0u32.to_be_bytes()); + wire.extend_from_slice(&2u32.to_be_bytes()); + wire.extend_from_slice(&1u32.to_be_bytes()); + wire.extend_from_slice(&5u16.to_be_bytes()); + wire.extend_from_slice(&[ + 3, b'w', b'w', b'w', 7, b'e', b'x', b'a', b'm', b'p', b'l', b'e', 3, b'c', b'o', b'm', 0, - 0, - 2, - 1, - 5, - Name::from_str("www.example.com").unwrap(), - vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 29, 31, - ], - ))); - let rdata_text = format_rdata(&rdata); - assert!(rdata_text.is_ok()); - if let Ok((parsed, raw_rdata)) = rdata_text { - assert!(raw_rdata.is_none()); - assert_eq!( - "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHR8=", - parsed.unwrap() - ); - } + ]); + wire.extend(0u8..=31); + test_format_rdata( + &BASE64.encode(&wire), + 24, + "NULL 8 0 0 2 1 5 www.example.com. AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + ); } #[test] @@ -1918,26 +1926,26 @@ mod tests { // so there isn't really a great way to reduce code duplication here. #[test] fn test_format_rdata_for_rsig_type() { - let rdata = RData::DNSSEC(DNSSECRData::RRSIG(RRSIG::new( - RecordType::NULL, - DNSSEC_Algorithm::RSASHA256, - 0, - 0, - 2, - 1, - 5, - Name::from_str("www.example.com").unwrap(), - vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 29, 31, - ], + let input = SigInput { + type_covered: RecordType::NULL, + algorithm: DNSSEC_Algorithm::RSASHA256, + num_labels: 0, + original_ttl: 0, + sig_expiration: SerialNumber::new(2), + sig_inception: SerialNumber::new(1), + key_tag: 5, + signer_name: Name::from_str("www.example.com").unwrap(), + }; + let rdata = RData::DNSSEC(DNSSECRData::RRSIG(RRSIG::from_sig( + input, + (0u8..=31).collect(), ))); let rdata_text = format_rdata(&rdata); assert!(rdata_text.is_ok()); if let Ok((parsed, raw_rdata)) = rdata_text { assert!(raw_rdata.is_none()); assert_eq!( - "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHR8=", + "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", parsed.unwrap() ); } diff --git a/lib/dnsmsg-parser/src/ede.rs b/lib/dnsmsg-parser/src/ede.rs index 0c3ee6c2f12e6..26bd22ed008ea 100644 --- a/lib/dnsmsg-parser/src/ede.rs +++ b/lib/dnsmsg-parser/src/ede.rs @@ -1,6 +1,6 @@ use hickory_proto::{ ProtoError, - serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder}, + serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder, DecodeError}, }; pub const EDE_OPTION_CODE: u16 = 15u16; @@ -77,14 +77,15 @@ impl BinEncodable for EDE { } impl<'a> BinDecodable<'a> for EDE { - fn read(decoder: &mut BinDecoder<'a>) -> Result { + fn read(decoder: &mut BinDecoder<'a>) -> Result { let info_code = decoder.read_u16()?.unverified(); let extra_text = if decoder.is_empty() { None } else { - Some(String::from_utf8( - decoder.read_vec(decoder.len())?.unverified(), - )?) + Some( + String::from_utf8(decoder.read_vec(decoder.len())?.unverified()) + .map_err(DecodeError::Utf8)?, + ) }; Ok(Self { info_code, From 6c3116a6e0a1d44113e5e2ce0b7e5aeeef3db785 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 5 May 2026 09:33:38 -0400 Subject: [PATCH 146/364] chore(ci): retry apt fetches in deb-verify to reduce flakes (#25367) Set Acquire::Retries=5 and Acquire::http::Timeout=30 on the apt-get calls in the deb-verify job so transient connection failures to the Ubuntu mirrors don't fail the whole job. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4f82bd06e3894..7e8455b60de58 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -204,8 +204,8 @@ jobs: image: ${{ matrix.container }} steps: - run: | - apt-get update && \ - apt-get install -y \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \ ca-certificates \ curl \ git \ From 95756d72356406d1a71625dbcc8f83e49f43947d Mon Sep 17 00:00:00 2001 From: jh7459-gh <90462481+jh7459-gh@users.noreply.github.com> Date: Tue, 5 May 2026 10:34:08 -0500 Subject: [PATCH 147/364] enhancement(transforms): dynamic rate for sample (#25035) feat(sample transform): add ratio_field and rate_field for dynamic sampling Add per-event dynamic sampling to the `sample` transform. `ratio_field` reads a numeric ratio in `(0, 1]` from the event; `rate_field` reads a positive integer rate (`1/N`). Either falls back to the static `rate`/`ratio` configuration when the field is missing or invalid. - `ratio_field` and `rate_field` are mutually exclusive, and neither can combine with `key_field` (key-based coherence conflicts with per-event values). - Sampling is event-granular and uses a hash of (group_by_key, counter) so mixed dynamic values within the same `group_by` group sample at their respective expected rates. - `group_by` is supported alongside dynamic fields for independent per-group sampling. - The dynamic-rate path is `NonZeroU64`-typed end-to-end so the `is_multiple_of(0)` panic is unreachable by construction. Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.7 (1M context) --- .../sample_dynamic_ratio_field.feature.md | 3 + src/transforms/sample/config.rs | 195 ++++++++++- src/transforms/sample/tests.rs | 329 +++++++++++++++++- src/transforms/sample/transform.rs | 254 +++++++++++--- .../transforms/generated/sample.cue | 29 ++ 5 files changed, 749 insertions(+), 61 deletions(-) create mode 100644 changelog.d/sample_dynamic_ratio_field.feature.md diff --git a/changelog.d/sample_dynamic_ratio_field.feature.md b/changelog.d/sample_dynamic_ratio_field.feature.md new file mode 100644 index 0000000000000..d2e3c04839c19 --- /dev/null +++ b/changelog.d/sample_dynamic_ratio_field.feature.md @@ -0,0 +1,3 @@ +Added `ratio_field` and `rate_field` options to the `sample` transform to support dynamic per-event sampling, while requiring static `rate` or `ratio` fallback configuration and disallowing `ratio_field` and `rate_field` together. + +authors: jhammer diff --git a/src/transforms/sample/config.rs b/src/transforms/sample/config.rs index 209d2d655e7c9..1b5f7426d86a3 100644 --- a/src/transforms/sample/config.rs +++ b/src/transforms/sample/config.rs @@ -6,7 +6,7 @@ use vector_lib::{ }; use vrl::value::Kind; -use super::transform::{Sample, SampleMode}; +use super::transform::{DynamicSampleFields, Sample, SampleMode}; use crate::{ conditions::AnyCondition, config::{ @@ -29,10 +29,23 @@ pub enum SampleError { #[snafu(display("Only non-zero numbers are allowed values for `rate`"))] InvalidRate, + #[snafu(display("Only one value can be provided for either 'rate' or 'ratio', but not both"))] + InvalidStaticConfiguration, + + #[snafu(display( + "Only one value can be provided for either 'ratio_field' or 'rate_field', but not both" + ))] + InvalidDynamicConfiguration, + + #[snafu(display( + "Exactly one value must be provided for either 'rate' or 'ratio' to configure static sampling" + ))] + MissingStaticConfiguration, + #[snafu(display( - "Exactly one value must be provided for either 'rate' or 'ratio', but not both" + "'key_field' cannot be combined with 'ratio_field' or 'rate_field' because dynamic values can vary per event and break key-based coherence" ))] - InvalidConfiguration, + InvalidKeyFieldDynamicCombination, } /// Configuration for the `sample` transform. @@ -61,6 +74,24 @@ pub struct SampleConfig { #[configurable(validation(range(min = 0.0, max = 1.0)))] pub ratio: Option, + /// The event field whose numeric value is used as the sampling ratio on a per-event basis. + /// + /// Accepts integer, floating point, or string values that parse as a number. The value must be + /// in `(0, 1]` to be considered valid (for example, `0.25` keeps 25%). If the field is missing + /// or invalid, static sampling settings (`rate` or `ratio`) are used as a fallback. + /// This option cannot be used together with `rate_field`. + #[configurable(metadata(docs::examples = "sample_rate"))] + pub ratio_field: Option, + + /// The event field whose integer value is used as the sampling rate on a per-event basis, expressed as `1/N`. + /// + /// Accepts an integer, or a string that parses as a positive integer; floating point values + /// are rejected. The value must be a positive integer to be considered valid. If the field is + /// missing or invalid, static sampling settings (`rate` or `ratio`) are used as a fallback. + /// This option cannot be used together with `ratio_field`. + #[configurable(metadata(docs::examples = "sample_rate_n"))] + pub rate_field: Option, + /// The name of the field whose value is hashed to determine if the event should be /// sampled. /// @@ -72,6 +103,8 @@ pub struct SampleConfig { /// /// This can be useful to, for example, ensure that all logs for a given transaction are /// sampled together, but that overall `1/N` transactions are sampled. + /// + /// This option cannot be combined with `ratio_field` or `rate_field`. #[configurable(metadata(docs::examples = "message"))] pub key_field: Option, @@ -84,6 +117,9 @@ pub struct SampleConfig { /// /// If left unspecified, or if the event doesn't have `group_by`, then the event is not /// sampled separately. + /// + /// This can also be used with `ratio_field` or `rate_field` to apply dynamic sampling + /// independently per rendered group value. #[configurable(metadata( docs::examples = "{{ service }}", docs::examples = "{{ hostname }}-{{ service }}" @@ -96,6 +132,18 @@ pub struct SampleConfig { impl SampleConfig { fn sample_rate(&self) -> Result { + if self.ratio_field.is_some() && self.rate_field.is_some() { + return Err(SampleError::InvalidDynamicConfiguration); + } + + if self.key_field.is_some() && (self.ratio_field.is_some() || self.rate_field.is_some()) { + return Err(SampleError::InvalidKeyFieldDynamicCombination); + } + + if self.rate.is_some() && self.ratio.is_some() { + return Err(SampleError::InvalidStaticConfiguration); + } + match (self.rate, self.ratio) { (None, Some(ratio)) => { if ratio <= 0.0 { @@ -111,7 +159,8 @@ impl SampleConfig { Ok(SampleMode::new_rate(rate)) } } - _ => Err(SampleError::InvalidConfiguration), + (None, None) => Err(SampleError::MissingStaticConfiguration), + _ => Err(SampleError::InvalidStaticConfiguration), } } } @@ -121,6 +170,8 @@ impl GenerateConfig for SampleConfig { toml::Value::try_from(Self { rate: None, ratio: Some(0.1), + ratio_field: None, + rate_field: None, key_field: None, group_by: None, exclude: None::, @@ -134,19 +185,37 @@ impl GenerateConfig for SampleConfig { #[typetag::serde(name = "sample")] impl TransformConfig for SampleConfig { async fn build(&self, context: &TransformContext) -> crate::Result { - Ok(Transform::function(Sample::new( - Self::NAME.to_string(), - self.sample_rate()?, - self.key_field.clone(), - self.group_by.clone(), - self.exclude - .as_ref() - .map(|condition| { - condition.build(&context.enrichment_tables, &context.metrics_storage) - }) - .transpose()?, - self.sample_rate_key.clone(), - ))) + let sample_mode = self.sample_rate()?; + let exclude = self + .exclude + .as_ref() + .map(|condition| condition.build(&context.enrichment_tables, &context.metrics_storage)) + .transpose()?; + + let sample = if self.ratio_field.is_some() || self.rate_field.is_some() { + Sample::new_with_dynamic( + Self::NAME.to_string(), + sample_mode, + DynamicSampleFields { + ratio_field: self.ratio_field.clone(), + rate_field: self.rate_field.clone(), + }, + self.group_by.clone(), + exclude, + self.sample_rate_key.clone(), + ) + } else { + Sample::new( + Self::NAME.to_string(), + sample_mode, + self.key_field.clone(), + self.group_by.clone(), + exclude, + self.sample_rate_key.clone(), + ) + }; + + Ok(Transform::function(sample)) } fn input(&self) -> Input { @@ -191,10 +260,100 @@ pub fn default_sample_rate_key() -> OptionalValuePath { #[cfg(test)] mod tests { - use crate::transforms::sample::config::SampleConfig; + use crate::{ + config::TransformConfig, + transforms::sample::config::{SampleConfig, SampleError}, + }; #[test] fn generate_config() { crate::test_util::test_generate_config::(); } + + #[test] + fn rejects_dynamic_ratio_only_configuration() { + let config = SampleConfig { + rate: None, + ratio: None, + ratio_field: Some("sample_rate".to_string()), + rate_field: None, + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::MissingStaticConfiguration)); + } + + #[test] + fn rejects_dynamic_rate_only_configuration() { + let config = SampleConfig { + rate: None, + ratio: None, + ratio_field: None, + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::MissingStaticConfiguration)); + } + + #[test] + fn validates_static_with_dynamic_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: None, + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + assert!(config.validate(&crate::schema::Definition::any()).is_ok()); + } + + #[test] + fn rejects_both_dynamic_fields_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: Some("sample_rate".to_string()), + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::InvalidDynamicConfiguration)); + } + + #[test] + fn rejects_key_field_with_dynamic_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: Some("sample_ratio".to_string()), + rate_field: None, + key_field: Some("trace_id".to_string()), + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!( + err, + SampleError::InvalidKeyFieldDynamicCombination + )); + } } diff --git a/src/transforms/sample/tests.rs b/src/transforms/sample/tests.rs index c705ff2cc4f1f..e6fec5c12a35d 100644 --- a/src/transforms/sample/tests.rs +++ b/src/transforms/sample/tests.rs @@ -1,4 +1,5 @@ use approx::assert_relative_eq; +use indoc::indoc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use vector_lib::lookup::lookup_v2::OptionalValuePath; @@ -14,7 +15,7 @@ use crate::{ FunctionTransform, OutputBuffer, sample::{ config::{SampleConfig, default_sample_rate_key}, - transform::{Sample, SampleMode}, + transform::{DynamicSampleFields, Sample, SampleMode}, }, test::{create_topology, transform_one}, }, @@ -26,6 +27,8 @@ async fn emits_internal_events() { let config = SampleConfig { rate: None, ratio: Some(1.0), + ratio_field: None, + rate_field: None, key_field: None, group_by: None, exclude: None, @@ -324,6 +327,330 @@ fn sample_at_rates_higher_then_half() { } } +#[test] +fn dynamic_ratio_field_overrides_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.1), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_ratio", 1.0); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_ratio_field_falls_back_to_static_ratio_when_missing() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + None, + None, + default_sample_rate_key(), + ); + + let event = Event::Log(LogEvent::from("hello")); + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_overrides_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_rate", 1); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_falls_back_to_static_ratio_when_missing() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let event = Event::Log(LogEvent::from("hello")); + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_rejects_float_and_falls_back_to_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_rate", 2.0); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_ratio_honors_group_by_key() { + let ratio = 0.5_f64; + let events_per_service = 200; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_service_a = 0; + let mut sampled_service_b = 0; + for _ in 0..events_per_service { + for service in ["service-a", "service-b"] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", service); + log.insert("dynamic_ratio", ratio); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], "0.5".into()); + if service == "service-a" { + sampled_service_a += 1; + } else { + sampled_service_b += 1; + } + } + } + } + + assert!( + (60..140).contains(&sampled_service_a), + "service-a sampled {} out of {events_per_service}", + sampled_service_a + ); + assert!( + (60..140).contains(&sampled_service_b), + "service-b sampled {} out of {events_per_service}", + sampled_service_b + ); +} + +#[test] +fn dynamic_rate_honors_group_by_key() { + let rate = 2_i64; + let events_per_service = 200; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_service_a = 0; + let mut sampled_service_b = 0; + for _ in 0..events_per_service { + for service in ["service-a", "service-b"] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", service); + log.insert("dynamic_rate", rate); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], "2".into()); + if service == "service-a" { + sampled_service_a += 1; + } else { + sampled_service_b += 1; + } + } + } + } + + assert!( + (60..140).contains(&sampled_service_a), + "service-a sampled {} out of {events_per_service}", + sampled_service_a + ); + assert!( + (60..140).contains(&sampled_service_b), + "service-b sampled {} out of {events_per_service}", + sampled_service_b + ); +} + +#[test] +fn dynamic_ratio_group_by_samples_mixed_ratios_at_expected_rates() { + let events_per_ratio = 500; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_low_ratio = 0; + let mut sampled_high_ratio = 0; + for _ in 0..events_per_ratio { + for (ratio, is_low_ratio) in [(0.25_f64, true), (0.75_f64, false)] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", "service-a"); + log.insert("dynamic_ratio", ratio); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], ratio.to_string().into()); + if is_low_ratio { + sampled_low_ratio += 1; + } else { + sampled_high_ratio += 1; + } + } + } + } + + assert!( + (80..220).contains(&sampled_low_ratio), + "ratio=0.25 sampled {} out of {events_per_ratio}", + sampled_low_ratio + ); + assert!( + (300..450).contains(&sampled_high_ratio), + "ratio=0.75 sampled {} out of {events_per_ratio}", + sampled_high_ratio + ); + assert!( + sampled_high_ratio > sampled_low_ratio, + "ratio=0.75 sampled {sampled_high_ratio}, ratio=0.25 sampled {sampled_low_ratio}" + ); +} + +#[test] +fn dynamic_rate_group_by_samples_mixed_rates_at_expected_rates() { + let events_per_rate = 600; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_rate_2 = 0; + let mut sampled_rate_3 = 0; + for _ in 0..events_per_rate { + for rate in [2_i64, 3_i64] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", "service-a"); + log.insert("dynamic_rate", rate); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], rate.to_string().into()); + if rate == 2 { + sampled_rate_2 += 1; + } else { + sampled_rate_3 += 1; + } + } + } + } + + assert!( + (220..380).contains(&sampled_rate_2), + "rate=2 sampled {} out of {events_per_rate}", + sampled_rate_2 + ); + assert!( + (120..280).contains(&sampled_rate_3), + "rate=3 sampled {} out of {events_per_rate}", + sampled_rate_3 + ); + assert!( + sampled_rate_2 > sampled_rate_3, + "rate=2 sampled {sampled_rate_2}, rate=3 sampled {sampled_rate_3}" + ); +} + +#[tokio::test] +async fn dynamic_field_config_drives_dynamic_sampling() { + assert_transform_compliance(async move { + let config: SampleConfig = serde_yaml::from_str(indoc! {r#" + ratio_field: dynamic_ratio + ratio: 1.0 + "#}) + .expect("config should deserialize"); + + let (tx, rx) = mpsc::channel(1); + let (topology, mut out) = create_topology(ReceiverStream::new(rx), config).await; + + let mut log = LogEvent::from("hello"); + log.insert("dynamic_ratio", 1.0); + tx.send(log.into()).await.unwrap(); + + let event = out.recv().await.expect("event should be sampled"); + assert_eq!(event.as_log()["sample_rate"], "1".into()); + + drop(tx); + topology.stop().await; + assert_eq!(out.recv().await, None); + }) + .await +} + fn condition_contains(key: &str, needle: &str) -> Condition { let vrl_config = VrlConfig { source: format!(r#"contains!(."{key}", "{needle}")"#), diff --git a/src/transforms/sample/transform.rs b/src/transforms/sample/transform.rs index 74786a43fdc51..00728b076e68c 100644 --- a/src/transforms/sample/transform.rs +++ b/src/transforms/sample/transform.rs @@ -1,4 +1,9 @@ -use std::{collections::HashMap, fmt}; +use std::{ + collections::HashMap, + fmt, + hash::{Hash, Hasher}, + num::NonZeroU64, +}; use vector_lib::{ config::LegacyKey, @@ -91,6 +96,26 @@ impl SampleMode { } } +enum EventSampleMode { + Ratio(f64), + Rate(NonZeroU64), +} + +impl EventSampleMode { + fn sample_rate_label(&self) -> String { + match self { + Self::Ratio(ratio) => ratio.to_string(), + Self::Rate(rate) => rate.to_string(), + } + } +} + +#[derive(Clone, Default)] +pub struct DynamicSampleFields { + pub ratio_field: Option, + pub rate_field: Option, +} + impl fmt::Display for SampleMode { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // Avoids the print of an additional '.0' which was not performed in the previous @@ -102,12 +127,24 @@ impl fmt::Display for SampleMode { } } +#[derive(Clone)] +pub enum SampleKeySource { + Static { + key_field: Option, + group_by: Option