From ff11a8b5c19d3ad030c057c0240426a9f099eb06 Mon Sep 17 00:00:00 2001 From: Vamshikrishna Monagari Date: Sun, 8 Mar 2026 23:05:42 -0400 Subject: [PATCH 1/3] Alert through logs for dropped spans in BatchSpanProcessor --- .../sdk/trace/export/BatchSpanProcessor.java | 7 +++++ .../trace/export/BatchSpanProcessorTest.java | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java b/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java index 2febbb46667..4fb4302329e 100644 --- a/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java +++ b/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java @@ -11,6 +11,7 @@ import io.opentelemetry.sdk.common.InternalTelemetryVersion; import io.opentelemetry.sdk.common.internal.ComponentId; import io.opentelemetry.sdk.common.internal.DaemonThreadFactory; +import io.opentelemetry.sdk.common.internal.ThrottlingLogger; import io.opentelemetry.sdk.common.internal.ThrowableUtil; import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; @@ -46,6 +47,7 @@ public final class BatchSpanProcessor implements SpanProcessor { ComponentId.generateLazy("batching_span_processor"); private static final Logger logger = Logger.getLogger(BatchSpanProcessor.class.getName()); + private static final ThrottlingLogger throttledLogger = new ThrottlingLogger(logger); private static final String WORKER_THREAD_NAME = BatchSpanProcessor.class.getSimpleName() + "_WorkerThread"; @@ -212,6 +214,11 @@ private void addSpan(ReadableSpan span) { spanProcessorInstrumentation.buildQueueMetricsOnce(maxQueueSize, queue::size); if (!queue.offer(span)) { spanProcessorInstrumentation.dropSpans(1); + throttledLogger.log( + Level.WARNING, + "BatchSpanProcessor dropped a span because the queue is full (maxQueueSize=" + + maxQueueSize + + ")"); } else { if (queueSize.incrementAndGet() >= spansNeeded.get()) { signal.offer(true); diff --git a/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java b/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java index 98bc45705b3..fecf7005f18 100644 --- a/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java +++ b/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java @@ -21,6 +21,7 @@ import io.opentelemetry.api.internal.GuardedBy; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; +import io.github.netmikey.logunit.api.LogCapturer; import io.opentelemetry.internal.testing.slf4j.SuppressLogger; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.trace.ReadableSpan; @@ -29,6 +30,7 @@ import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.time.Duration; +import org.slf4j.event.Level; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -42,6 +44,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -62,9 +65,13 @@ class BatchSpanProcessorTest { @Mock private Sampler mockSampler; @Mock private SpanExporter mockSpanExporter; + @RegisterExtension + LogCapturer logs = LogCapturer.create().captureForType(BatchSpanProcessor.class); + @BeforeEach void setUp() { when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofSuccess()); + when(mockSpanExporter.export(anyList())).thenReturn(CompletableResultCode.ofSuccess()); } @AfterEach @@ -232,6 +239,30 @@ void exportMoreSpansThanTheBufferSize() { span6.toSpanData())); } + @Test + void droppedSpanIsLogged() { + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + BatchSpanProcessor.builder(mockSpanExporter) + .setMaxQueueSize(1) + .setMaxExportBatchSize(1_000) + .setScheduleDelay(Duration.ofDays(1)) + .build()) + .build(); + + // Add two spans quickly to trigger a drop when the queue is full. + createEndedSpan(SPAN_NAME_1); + createEndedSpan(SPAN_NAME_2); + + await() + .untilAsserted( + () -> + logs.assertContains( + loggingEvent -> loggingEvent.getLevel().equals(Level.WARN), + "BatchSpanProcessor dropped a span")); + } + @Test void forceExport() { WaitingSpanExporter waitingSpanExporter = From 912e9973f216c01172334806451d627fb3b5c589 Mon Sep 17 00:00:00 2001 From: Vamshikrishna Monagari Date: Tue, 10 Mar 2026 21:27:45 -0400 Subject: [PATCH 2/3] Fix variable name for throttling logger in BatchSpanProcessor --- .../io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java b/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java index 4fb4302329e..047bf359c89 100644 --- a/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java +++ b/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java @@ -47,7 +47,7 @@ public final class BatchSpanProcessor implements SpanProcessor { ComponentId.generateLazy("batching_span_processor"); private static final Logger logger = Logger.getLogger(BatchSpanProcessor.class.getName()); - private static final ThrottlingLogger throttledLogger = new ThrottlingLogger(logger); + private static final ThrottlingLogger throttlingLogger = new ThrottlingLogger(logger); private static final String WORKER_THREAD_NAME = BatchSpanProcessor.class.getSimpleName() + "_WorkerThread"; @@ -214,7 +214,7 @@ private void addSpan(ReadableSpan span) { spanProcessorInstrumentation.buildQueueMetricsOnce(maxQueueSize, queue::size); if (!queue.offer(span)) { spanProcessorInstrumentation.dropSpans(1); - throttledLogger.log( + throttlingLogger.log( Level.WARNING, "BatchSpanProcessor dropped a span because the queue is full (maxQueueSize=" + maxQueueSize From d14a2cca0a875abc714ea2ebd0ea96ef3e3be0ec Mon Sep 17 00:00:00 2001 From: Vamshikrishna Monagari Date: Tue, 17 Mar 2026 20:32:35 -0400 Subject: [PATCH 3/3] Replace ThrottlingLogger with per-batch drop count logging in BatchSpanProcessor Instead of logging a throttled message per dropped span, accumulate drop counts using an AtomicInteger and report the total number of dropped spans once per export cycle in exportCurrentBatch(). This gives users visibility into the magnitude of span drops, not just that drops are occurring. The counter is incremented atomically in addSpan() (called from application threads) and read/reset with getAndSet(0) in exportCurrentBatch() (called from the worker thread). --- .../sdk/trace/export/BatchSpanProcessor.java | 21 ++++++++++++------- .../trace/export/BatchSpanProcessorTest.java | 6 +++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java b/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java index 047bf359c89..f264128696e 100644 --- a/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java +++ b/sdk/trace/src/main/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessor.java @@ -11,7 +11,6 @@ import io.opentelemetry.sdk.common.InternalTelemetryVersion; import io.opentelemetry.sdk.common.internal.ComponentId; import io.opentelemetry.sdk.common.internal.DaemonThreadFactory; -import io.opentelemetry.sdk.common.internal.ThrottlingLogger; import io.opentelemetry.sdk.common.internal.ThrowableUtil; import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; @@ -47,7 +46,6 @@ public final class BatchSpanProcessor implements SpanProcessor { ComponentId.generateLazy("batching_span_processor"); private static final Logger logger = Logger.getLogger(BatchSpanProcessor.class.getName()); - private static final ThrottlingLogger throttlingLogger = new ThrottlingLogger(logger); private static final String WORKER_THREAD_NAME = BatchSpanProcessor.class.getSimpleName() + "_WorkerThread"; @@ -186,6 +184,7 @@ private static final class Worker implements Runnable { private volatile boolean continueWork = true; private final ArrayList batch; private final long maxQueueSize; + private final AtomicInteger droppedSpanCount = new AtomicInteger(0); private Worker( SpanExporter spanExporter, @@ -214,11 +213,7 @@ private void addSpan(ReadableSpan span) { spanProcessorInstrumentation.buildQueueMetricsOnce(maxQueueSize, queue::size); if (!queue.offer(span)) { spanProcessorInstrumentation.dropSpans(1); - throttlingLogger.log( - Level.WARNING, - "BatchSpanProcessor dropped a span because the queue is full (maxQueueSize=" - + maxQueueSize - + ")"); + droppedSpanCount.incrementAndGet(); } else { if (queueSize.incrementAndGet() >= spansNeeded.get()) { signal.offer(true); @@ -322,6 +317,18 @@ private void exportCurrentBatch() { return; } + int dropped = droppedSpanCount.getAndSet(0); + if (dropped > 0) { + logger.log( + Level.WARNING, + "BatchSpanProcessor dropped " + + dropped + + " span(s) since the last export because the queue is full" + + " (maxQueueSize=" + + maxQueueSize + + ")"); + } + String error = null; try { CompletableResultCode result = spanExporter.export(Collections.unmodifiableList(batch)); diff --git a/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java b/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java index fecf7005f18..d64b4604b6c 100644 --- a/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java +++ b/sdk/trace/src/test/java/io/opentelemetry/sdk/trace/export/BatchSpanProcessorTest.java @@ -246,8 +246,8 @@ void droppedSpanIsLogged() { .addSpanProcessor( BatchSpanProcessor.builder(mockSpanExporter) .setMaxQueueSize(1) - .setMaxExportBatchSize(1_000) - .setScheduleDelay(Duration.ofDays(1)) + .setMaxExportBatchSize(1) + .setScheduleDelay(Duration.ofMillis(1)) .build()) .build(); @@ -260,7 +260,7 @@ void droppedSpanIsLogged() { () -> logs.assertContains( loggingEvent -> loggingEvent.getLevel().equals(Level.WARN), - "BatchSpanProcessor dropped a span")); + "BatchSpanProcessor dropped")); } @Test