diff --git a/changelog.d/host_metrics_oom_kill.feature.md b/changelog.d/host_metrics_oom_kill.feature.md new file mode 100644 index 0000000000000..5f97113814431 --- /dev/null +++ b/changelog.d/host_metrics_oom_kill.feature.md @@ -0,0 +1,4 @@ +The `host_metrics` source now exposes an `oom_kill` counter metric on Linux, +reporting the number of Out-Of-Memory kill events recorded by the kernel. + +authors: simonhammes diff --git a/src/sources/host_metrics/memory.rs b/src/sources/host_metrics/memory.rs index fad4e58fc853d..65ffccfc188ba 100644 --- a/src/sources/host_metrics/memory.rs +++ b/src/sources/host_metrics/memory.rs @@ -10,6 +10,9 @@ use vector_lib::event::MetricTags; use super::HostMetrics; use crate::internal_events::HostMetricsScrapeDetailError; +#[cfg(target_os = "linux")] +const OOM_KILL: &str = "oom_kill"; + impl HostMetrics { pub async fn memory_metrics(&self, output: &mut super::MetricsBuffer) { output.name = "memory"; @@ -122,4 +125,62 @@ impl HostMetrics { } } } + + #[cfg(target_os = "linux")] + pub async fn vmstat_metrics(&self, output: &mut super::MetricsBuffer) { + output.name = "memory"; + + // Spawn blocking task to avoid blocking the async runtime with synchronous I/O + let result = tokio::task::spawn_blocking(procfs::vmstat) + .await + .unwrap_or_else(|join_error| { + Err(procfs::ProcError::Other(format!( + "Failed to join blocking task: {}", + join_error + ))) + }); + + match result { + Ok(stats) => { + if let Some(&oom_kill) = stats.get("oom_kill") { + output.counter(OOM_KILL, oom_kill as f64, MetricTags::default()); + } + } + Err(error) => { + emit!(HostMetricsScrapeDetailError { + message: "Failed to load vmstat info.", + error, + }); + } + } + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use crate::event::metric::MetricValue; + use crate::sources::host_metrics::{HostMetrics, HostMetricsConfig, MetricsBuffer}; + + use super::OOM_KILL; + + #[tokio::test] + async fn generates_vmstat_oom_kill_metric() { + let mut buffer = MetricsBuffer::new(None); + HostMetrics::new(HostMetricsConfig::default()) + .vmstat_metrics(&mut buffer) + .await; + let metrics = buffer.metrics; + + assert_eq!(metrics.len(), 1); + + let metric = &metrics[0]; + assert_eq!(metric.name(), OOM_KILL); + assert!( + matches!(metric.value(), MetricValue::Counter { .. }), + "oom_kill metric should be a counter" + ); + + let tags = metric.tags().expect("metric must have tags"); + assert_eq!(tags.get("collector"), Some("memory")); + } } diff --git a/src/sources/host_metrics/mod.rs b/src/sources/host_metrics/mod.rs index c3e8075a4b110..17a8cd9db9681 100644 --- a/src/sources/host_metrics/mod.rs +++ b/src/sources/host_metrics/mod.rs @@ -423,6 +423,8 @@ impl HostMetrics { if self.config.has_collector(Collector::Memory) { self.memory_metrics(&mut buffer).await; self.swap_metrics(&mut buffer).await; + #[cfg(target_os = "linux")] + self.vmstat_metrics(&mut buffer).await; } if self.config.has_collector(Collector::Network) { self.network_metrics(&mut buffer).await; diff --git a/website/cue/reference/components/sources/host_metrics.cue b/website/cue/reference/components/sources/host_metrics.cue index a63963f35c390..e985e7f6550f5 100644 --- a/website/cue/reference/components/sources/host_metrics.cue +++ b/website/cue/reference/components/sources/host_metrics.cue @@ -166,6 +166,7 @@ components: sources: host_metrics: { memory_total_bytes: _host & _memory_gauge & {description: "The total number of bytes of main memory."} memory_used_bytes: _host & _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} memory_wired_bytes: _host & _memory_macos & {description: "The number of wired bytes of main memory."} + oom_kill: _host & _memory_counter & _linux & {description: "The number of Out-Of-Memory (OOM) kill events recorded."} // Host network network_receive_bytes_total: _host & _network_counter & {description: "The number of bytes received on this interface."} @@ -177,15 +178,15 @@ components: sources: host_metrics: { network_transmit_packets_total: _host & _network_nomac & {description: "The number of packets transmitted on this interface."} // Host tcp - tcp_connections_total: _host & _tcp_linux & _tcp_gauge & {description: "The number of TCP connections."} - tcp_tx_queued_bytes_total: _host & _tcp_linux & { + tcp_connections_total: _host & _linux & _tcp_gauge & {description: "The number of TCP connections."} + tcp_tx_queued_bytes_total: _host & _linux & { description: "The number of bytes in the send queue across all connections." type: "gauge" tags: _host_metrics_tags & { collector: examples: ["tcp"] } } - tcp_rx_queued_bytes_total: _host & _tcp_linux & { + tcp_rx_queued_bytes_total: _host & _linux & { description: "The number of bytes in the receive queue across all connections." type: "gauge" tags: _host_metrics_tags & { @@ -300,7 +301,7 @@ components: sources: host_metrics: { } } - _tcp_linux: {relevant_when: "OS is Linux"} + _linux: {relevant_when: "OS is Linux"} _tcp_gauge: { type: "gauge" tags: _host_metrics_tags & {