From fa5a66d40b84405ffd0a56f7a420e438e72c07bf Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Wed, 5 Aug 2026 16:19:15 +1000 Subject: [PATCH 1/2] feat(collector): derive request metrics with trace exemplars from spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds spanmetricsconnector to the collector build and wires it into the generated OpAMP config behind ENABLE_SPAN_METRICS, off by default. The connector consumes the traces pipeline and feeds a dedicated metrics pipeline, so traces.span.metrics.* reach ClickHouse with Exemplars.* pointing back at the spans they came from. Exponential histogram buckets rather than a fixed ladder. An explicit ladder puts everything slow into one wide top bucket, so a high quantile interpolates past the slowest real request and no exemplar can ever sit on the plotted line. ENABLE_SPAN_METRICS_PROM_RW additionally remote-writes the derived metrics to a Prometheus endpoint. The endpoint is resolved API-side and inlined into the generated config, so the collector container does not need it in its own environment, and the flag stays off unless the endpoint is set. The connector key must be `spanmetrics` — that is the component type spanmetricsconnector registers. A config naming an unregistered type fails to decode as a whole, and docker/otel-collector/config.yaml supplies no pipelines of its own, so a typo here stops all ingestion rather than just disabling this feature. A test pins every generated component id against builder-config.yaml so that cannot ship again. This is deliberately separate from the UI that consumes these metrics: it changes what the collector runs and has its own rollout ordering. Roll the collector image out before enabling the flag. --- .changeset/spanmetrics-connector.md | 25 +++ .gitignore | 4 + packages/api/src/config.ts | 15 ++ .../__tests__/opampController.test.ts | 150 ++++++++++++++++++ .../src/opamp/controllers/opampController.ts | 82 ++++++++++ packages/otel-collector/builder-config.yaml | 3 + 6 files changed, 279 insertions(+) create mode 100644 .changeset/spanmetrics-connector.md diff --git a/.changeset/spanmetrics-connector.md b/.changeset/spanmetrics-connector.md new file mode 100644 index 0000000000..53d3ce7bd6 --- /dev/null +++ b/.changeset/spanmetrics-connector.md @@ -0,0 +1,25 @@ +--- +'@hyperdx/otel-collector': minor +'@hyperdx/api': minor +--- + +feat: derive request metrics with trace exemplars from spans + +Adds the OpenTelemetry `spanmetricsconnector` to the collector build and wires it +into the generated OpAMP config behind `ENABLE_SPAN_METRICS`, off by default. The +connector reads the traces pipeline and feeds a dedicated metrics pipeline, so +`traces.span.metrics.*` land in ClickHouse with `Exemplars.*` pointing back at the +spans they were measured from. + +Histogram buckets are exponential rather than a fixed ladder: an explicit ladder +puts everything slow into one wide top bucket, so a high quantile interpolates well +past the slowest real request and no exemplar can sit on the plotted line. + +`ENABLE_SPAN_METRICS_PROM_RW` additionally remote-writes the derived metrics to a +Prometheus endpoint, for exercising Prometheus's native exemplar path. + +Requires a collector built from the current `builder-config.yaml`. Roll the +collector image out before enabling the flag: a config naming a component type the +binary does not register fails to decode as a whole, and the shipped +`docker/otel-collector/config.yaml` defines no pipelines of its own, so a rejected +remote config leaves the collector with nothing to run. diff --git a/.gitignore b/.gitignore index 31c6b70198..1f3ae85a33 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,7 @@ docker-compose.prod.yml # webstorm .idea + +# Stryker mutation-testing output +packages/*/reports/ +packages/*/.stryker-tmp/ diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 7c6a242969..28da4e564b 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -57,6 +57,21 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; +// Opt-in: have the collector derive request metrics (with trace exemplars) from +// spans via the spanmetrics connector. Off by default; enabled in dev so the +// telemetry-generator's traces produce coherent metric exemplars end-to-end. +export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; + +// Opt-in: also remote-write the span-derived metrics (with exemplars) to a +// Prometheus endpoint so the native Prometheus query_exemplars path can be +// tested against real data. The endpoint is resolved here (API side) and +// inlined into the generated collector config, so the collector container does +// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the +// endpoint to be set; without it the feature stays disabled. +export const SPAN_METRICS_PROM_RW_ENDPOINT = env.SPAN_METRICS_PROM_RW_ENDPOINT; +export const IS_SPAN_METRICS_PROM_RW_ENABLED = + env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT; + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts index f86baa8039..2a2f17833d 100644 --- a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts +++ b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts @@ -7,6 +7,9 @@ const configState = { INGESTION_API_KEY: '' as string, IS_PROMQL_ENABLED: false, ENABLE_DATADOG_RECEIVER: false, + IS_SPAN_METRICS_ENABLED: false, + IS_SPAN_METRICS_PROM_RW_ENABLED: false, + SPAN_METRICS_PROM_RW_ENDPOINT: undefined as string | undefined, }; jest.mock('@/config', () => ({ @@ -28,8 +31,20 @@ jest.mock('@/config', () => ({ get ENABLE_DATADOG_RECEIVER() { return configState.ENABLE_DATADOG_RECEIVER; }, + get IS_SPAN_METRICS_ENABLED() { + return configState.IS_SPAN_METRICS_ENABLED; + }, + get IS_SPAN_METRICS_PROM_RW_ENABLED() { + return configState.IS_SPAN_METRICS_PROM_RW_ENABLED; + }, + get SPAN_METRICS_PROM_RW_ENDPOINT() { + return configState.SPAN_METRICS_PROM_RW_ENDPOINT; + }, })); +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + import { buildOtelCollectorConfig } from '@/opamp/controllers/opampController'; const resetConfig = () => { @@ -39,6 +54,9 @@ const resetConfig = () => { configState.INGESTION_API_KEY = ''; configState.IS_PROMQL_ENABLED = false; configState.ENABLE_DATADOG_RECEIVER = false; + configState.IS_SPAN_METRICS_ENABLED = false; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = false; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = undefined; }; describe('opampController', () => { @@ -123,4 +141,136 @@ describe('opampController', () => { expect(cfg.service.extensions).toContain('bearertokenauth/hyperdx'); }); }); + + describe('buildOtelCollectorConfig span metrics', () => { + it('omits the span metrics connector when the flag is off (default)', () => { + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + expect(cfg.connectors?.spanmetrics).toBeUndefined(); + expect(cfg.service.pipelines['metrics/spanmetrics']).toBeUndefined(); + expect(cfg.service.pipelines.traces.exporters).not.toContain( + 'spanmetrics', + ); + }); + + it('derives span metrics as exponential histograms with exemplars', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + // Exponential rather than explicit buckets: a fixed ladder's wide top + // bucket makes high quantiles interpolate well past the slowest real + // request, so no exemplar can ever sit on the plotted line. + expect(cfg.connectors?.spanmetrics?.histogram).toEqual({ + unit: 'ms', + exponential: { max_size: 160 }, + }); + expect(cfg.connectors?.spanmetrics?.exemplars).toEqual({ + enabled: true, + }); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + ]); + // The connector has to be wired at both ends or it silently does nothing. + expect(cfg.service.pipelines.traces.exporters).toContain('spanmetrics'); + expect(cfg.service.pipelines['metrics/spanmetrics']?.receivers).toEqual([ + 'spanmetrics', + ]); + }); + + it('adds a remote-write exporter for the derived metrics when enabled', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = + 'http://prometheus:9090/api/v1/write'; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + expect( + cfg.exporters?.['prometheusremotewrite/spanmetrics'], + ).toMatchObject({ endpoint: 'http://prometheus:9090/api/v1/write' }); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + 'prometheusremotewrite/spanmetrics', + ]); + }); + + // A config naming a component type the binary doesn't register fails to + // decode as a whole, and docker/otel-collector/config.yaml defines no + // pipelines of its own — so a bad type name here leaves the collector with + // no usable config at all rather than just disabling one feature. Pin every + // generated component id against what builder-config.yaml actually builds. + it('only references component types the collector build registers', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = + 'http://prometheus:9090/write'; + configState.IS_PROMQL_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + const builderConfig = readFileSync( + resolve(__dirname, '../../../../../otel-collector/builder-config.yaml'), + 'utf8', + ); + // `/` ids share the base type, which is what gets registered. + const baseType = (id: string) => id.split('/')[0]; + // gomod lines end in `` — e.g. `.../connector/spanmetricsconnector` + // for type `spanmetrics`. Matching the suffixed forms is what makes + // `span_metrics` (the bug this test exists for) fail to resolve. + const registers = (id: string) => + ['connector', 'exporter', 'receiver', 'processor'].some(kind => + builderConfig.includes(`/${baseType(id)}${kind}`), + ); + + // Core components shipped in the collector binary rather than declared as + // contrib gomods in builder-config.yaml. + const CORE_COMPONENTS = ['nop', 'debug', 'memory_limiter', 'batch']; + + const declaredIds = [ + ...Object.keys(cfg.connectors ?? {}), + ...Object.keys(cfg.exporters ?? {}), + ...Object.keys(cfg.receivers ?? {}), + ]; + // Assert we actually looked at something — an empty set would make every + // loop below vacuously green, which is how a bad type name shipped. + expect(declaredIds.length).toBeGreaterThan(0); + + for (const id of declaredIds) { + if (CORE_COMPONENTS.includes(baseType(id))) continue; + expect(registers(id)).toBe(true); + } + + // Every pipeline reference must resolve to a declared component. Processors + // are included: they're named here but declared in + // docker/otel-collector/config.yaml, so they can't be checked against the + // declared set — but they still have to be types the build registers, since + // an unregistered processor fails the whole config to decode just like an + // unregistered connector. + const declared = new Set(declaredIds); + let pipelineRefs = 0; + for (const pipeline of Object.values(cfg.service.pipelines)) { + for (const id of [ + ...(pipeline?.receivers ?? []), + ...(pipeline?.exporters ?? []), + ]) { + expect(declared).toContain(id); + pipelineRefs++; + } + for (const id of pipeline?.processors ?? []) { + if (CORE_COMPONENTS.includes(baseType(id))) continue; + expect(registers(id)).toBe(true); + pipelineRefs++; + } + } + expect(pipelineRefs).toBeGreaterThan(0); + }); + }); }); diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 6dfb5f989c..67a9b77c5f 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -105,6 +105,13 @@ type CollectorConfig = { pipelines: string[]; }>; }; + spanmetrics?: { + histogram: { unit: string; exponential: { max_size: number } }; + dimensions: Array<{ name: string }>; + exemplars: { enabled: boolean }; + metrics_flush_interval: string; + namespace?: string; + }; }; exporters?: { nop?: null; @@ -155,6 +162,15 @@ type CollectorConfig = { enabled: boolean; }; }; + 'prometheusremotewrite/spanmetrics'?: { + endpoint: string; + tls: { + insecure: boolean; + }; + resource_to_telemetry_conversion: { + enabled: boolean; + }; + }; }; service: { extensions: string[]; @@ -345,6 +361,72 @@ export const buildOtelCollectorConfig = ( }; } + if ( + config.IS_SPAN_METRICS_ENABLED && + otelCollectorConfig.connectors && + otelCollectorConfig.exporters + ) { + // Derive request metrics (with trace exemplars) from spans. The connector + // consumes the traces pipeline and feeds a dedicated metrics pipeline, so + // the resulting `traces.span.metrics.*` land in ClickHouse with + // `Exemplars.*` pointing back at the spans they were measured from. + // + // The key MUST be `spanmetrics` — that is the component type + // spanmetricsconnector registers, and a config naming an unregistered type + // fails to decode wholesale. Because docker/otel-collector/config.yaml + // supplies no pipelines of its own, a rejected remote config leaves the + // collector with nothing to run, so a typo here takes ingestion down + // rather than just disabling this feature. + // + // Requires a collector built from the current builder-config.yaml (which + // added spanmetricsconnector). Enabling this flag against collectors built + // before that will have them reject the config for the same reason, so + // roll the collector image out first. + otelCollectorConfig.connectors.spanmetrics = { + histogram: { + unit: 'ms', + // Exponential (OTLP) / native (Prometheus) buckets rather than a fixed + // ladder. Explicit bounds put everything slow into one wide top bucket + // — with a 5s–10s bucket, a p99 interpolates towards 10s while the + // slowest real request was half that, and no exemplar can ever sit on + // the line. max_size is the connector's default; scale adapts to the + // observed range, giving sub-percent quantile error at any latency. + exponential: { max_size: 160 }, + }, + dimensions: [ + { name: 'http.route' }, + { name: 'http.method' }, + { name: 'host.region' }, + { name: 'app.tenant_id' }, + { name: 'http.status_code' }, + ], + exemplars: { enabled: true }, + metrics_flush_interval: '15s', + }; + otelCollectorConfig.service.pipelines.traces.exporters.push( + 'spanmetrics', + ); + + const spanMetricsExporters = ['clickhouse']; + // Optionally also remote-write the derived metrics (with exemplars) to a + // Prometheus endpoint, so the native Prometheus `query_exemplars` path can + // be exercised against the same real, generated data. + if (config.IS_SPAN_METRICS_PROM_RW_ENABLED) { + otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { + // Guaranteed set by IS_SPAN_METRICS_PROM_RW_ENABLED above. + endpoint: config.SPAN_METRICS_PROM_RW_ENDPOINT!, + tls: { insecure: true }, + resource_to_telemetry_conversion: { enabled: true }, + }; + spanMetricsExporters.push('prometheusremotewrite/spanmetrics'); + } + otelCollectorConfig.service.pipelines['metrics/spanmetrics'] = { + receivers: ['spanmetrics'], + processors: ['memory_limiter', 'batch'], + exporters: spanMetricsExporters, + }; + } + if (collectorAuthenticationEnforced) { if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) { // should never happen diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index ecd794a456..e84cad931a 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -135,6 +135,9 @@ connectors: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector + v__OTEL_COLLECTOR_VERSION__ extensions: # Core From 188f8180374e9b7d87d5516a2b67a6e27e130eaf Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Wed, 5 Aug 2026 22:16:57 +1000 Subject: [PATCH 2/2] fix(collector): bound span-metric cardinality and drop the bootstrap pipeline stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review findings on the spanmetrics connector. Cardinality. Dimensions dropped `app.tenant_id` and raw `host.region`, and now use the stable HTTP semconv spellings — `http.method`/`http.status_code` are pre-1.23 and absent from current SDKs. Added an `aggregation_cardinality_limit`, plus `resource_metrics_key_attributes`: the limit applies per resource-cache entry, so with the default key (every resource attribute, including per-pod ones) the real ceiling was the cache size times the limit, not the limit. Pipeline processors. The first attempt moved `processors:` into the bootstrap config to respect the rule in #2351, which was wrong: the supervisor merges the bootstrap config unconditionally, the pipeline only exists when the flag is on, and the collector rejects a pipeline with no receivers or exporters — failing the whole config. With the flag off, which is the default, every agent would have failed to start. Standalone mode would have failed unconditionally. Set inline instead, as `metrics/promql` already does, and pinned by a test that every bootstrap-declared pipeline is filled in by the generated config. Remote-write exporter. Dropped `tls.insecure`: on an HTTP exporter the URL scheme decides whether TLS is used and `insecure` only means anything to gRPC, so it was inert rather than weakening anything. Keyed the exporter off the endpoint rather than the flag, so a missing endpoint cannot emit a config that fails to decode. The generated config is served from the unauthenticated OpAMP endpoint, so an endpoint URL carrying credentials is now rejected, as are non-HTTP schemes. Turned off resource-to-telemetry conversion, which happens after the cardinality limit and would send host, pod and namespace to a third party. Also tightened the component-registration test to match the component kind rather than any kind, and documented the feature in the collector README. --- .changeset/spanmetrics-connector.md | 38 ++-- packages/api/src/__tests__/config.test.ts | 59 +++++ packages/api/src/config.ts | 31 ++- .../__tests__/opampController.test.ts | 173 ++++++++++++--- .../src/opamp/controllers/opampController.ts | 118 +++++++--- packages/otel-collector/README.md | 203 +++++++++++------- 6 files changed, 467 insertions(+), 155 deletions(-) diff --git a/.changeset/spanmetrics-connector.md b/.changeset/spanmetrics-connector.md index 53d3ce7bd6..685cca2d7f 100644 --- a/.changeset/spanmetrics-connector.md +++ b/.changeset/spanmetrics-connector.md @@ -5,21 +5,33 @@ feat: derive request metrics with trace exemplars from spans -Adds the OpenTelemetry `spanmetricsconnector` to the collector build and wires it -into the generated OpAMP config behind `ENABLE_SPAN_METRICS`, off by default. The -connector reads the traces pipeline and feeds a dedicated metrics pipeline, so -`traces.span.metrics.*` land in ClickHouse with `Exemplars.*` pointing back at the -spans they were measured from. +Adds the OpenTelemetry `spanmetricsconnector` to the collector build and wires +it into the generated OpAMP config behind `ENABLE_SPAN_METRICS`, off by default. +The connector reads the traces pipeline and feeds a dedicated metrics pipeline, +so `traces.span.metrics.*` land in ClickHouse with `Exemplars.*` pointing back +at the spans they were measured from. Histogram buckets are exponential rather than a fixed ladder: an explicit ladder -puts everything slow into one wide top bucket, so a high quantile interpolates well -past the slowest real request and no exemplar can sit on the plotted line. +puts everything slow into one wide top bucket, so a high quantile interpolates +well past the slowest real request and no exemplar can sit on the plotted line. -`ENABLE_SPAN_METRICS_PROM_RW` additionally remote-writes the derived metrics to a -Prometheus endpoint, for exercising Prometheus's native exemplar path. +Dimensions are limited to `http.route`, `http.request.method` and +`http.response.status_code` — each bounded by the application's route table or +by the HTTP spec — and the connector is given an explicit +`aggregation_cardinality_limit`. Temporality is cumulative, so nothing evicts a +series once created; one free-form dimension would grow collector memory and the +ClickHouse write volume without bound. + +`ENABLE_SPAN_METRICS_PROM_RW` additionally remote-writes the derived metrics to +a Prometheus endpoint, for exercising Prometheus's native exemplar path. Because +the generated config is served from the unauthenticated OpAMP endpoint, an +endpoint URL carrying `user:token@` credentials is rejected rather than inlined, +as are non-HTTP schemes. Resource attributes are not promoted to labels on this +exporter: that happens after the connector's cardinality limit and would send +host, pod and namespace to a third party. Requires a collector built from the current `builder-config.yaml`. Roll the -collector image out before enabling the flag: a config naming a component type the -binary does not register fails to decode as a whole, and the shipped -`docker/otel-collector/config.yaml` defines no pipelines of its own, so a rejected -remote config leaves the collector with nothing to run. +collector image out before enabling the flag: a config naming a component type +the binary does not register fails to decode as a whole, and the shipped +`docker/otel-collector/config.yaml` defines no pipelines of its own, so a +rejected remote config leaves the collector with nothing to run. diff --git a/packages/api/src/__tests__/config.test.ts b/packages/api/src/__tests__/config.test.ts index fedf486e0d..40c3ffcdae 100644 --- a/packages/api/src/__tests__/config.test.ts +++ b/packages/api/src/__tests__/config.test.ts @@ -59,4 +59,63 @@ describe('config', () => { }); }); }); + + describe('SPAN_METRICS_PROM_RW_ENDPOINT', () => { + const ORIGINAL = process.env.SPAN_METRICS_PROM_RW_ENDPOINT; + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.SPAN_METRICS_PROM_RW_ENDPOINT; + } else { + process.env.SPAN_METRICS_PROM_RW_ENDPOINT = ORIGINAL; + } + jest.resetModules(); + }); + + const resolve = (raw: string | undefined) => { + if (raw === undefined) { + delete process.env.SPAN_METRICS_PROM_RW_ENDPOINT; + } else { + process.env.SPAN_METRICS_PROM_RW_ENDPOINT = raw; + } + let resolved: string | undefined; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports, n/no-missing-require + resolved = require('@/config').SPAN_METRICS_PROM_RW_ENDPOINT; + }); + return resolved; + }; + + it('accepts a plain http or https endpoint', () => { + expect(resolve('http://prometheus:9090/api/v1/write')).toBe( + 'http://prometheus:9090/api/v1/write', + ); + expect(resolve('https://prom.example.com/api/v1/push')).toBe( + 'https://prom.example.com/api/v1/push', + ); + }); + + // The generated collector config is served from the unauthenticated OpAMP + // endpoint, so a credential in this URL is readable by anyone who can POST + // there. Pointing at a hosted Prometheus this way is the common case, which + // is why it is rejected rather than passed along. + it('rejects an endpoint carrying credentials', () => { + expect(resolve('https://user:token@prom.example.com/api/v1/push')).toBe( + undefined, + ); + expect(resolve('https://token@prom.example.com/api/v1/push')).toBe( + undefined, + ); + }); + + it('rejects a non-HTTP scheme or an unparseable URL', () => { + expect(resolve('file:///etc/passwd')).toBe(undefined); + expect(resolve('prometheus:9090')).toBe(undefined); + expect(resolve('not a url')).toBe(undefined); + }); + + it('is undefined when unset', () => { + expect(resolve(undefined)).toBe(undefined); + }); + }); }); diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 28da4e564b..85023e30ce 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -58,8 +58,14 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; // Opt-in: have the collector derive request metrics (with trace exemplars) from -// spans via the spanmetrics connector. Off by default; enabled in dev so the -// telemetry-generator's traces produce coherent metric exemplars end-to-end. +// spans via the spanmetrics connector. Off everywhere by default — set +// ENABLE_SPAN_METRICS=true to have the telemetry-generator's traces produce +// coherent metric exemplars end-to-end. +// +// Requires a collector built from the current builder-config.yaml. An older +// image does not register the `spanmetrics` type, and a config naming an +// unregistered type fails to decode as a whole, so the agent would fall back to +// no pipelines at all. Roll the collector image out before setting this. export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; // Opt-in: also remote-write the span-derived metrics (with exemplars) to a @@ -68,7 +74,26 @@ export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; // inlined into the generated collector config, so the collector container does // not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the // endpoint to be set; without it the feature stays disabled. -export const SPAN_METRICS_PROM_RW_ENDPOINT = env.SPAN_METRICS_PROM_RW_ENDPOINT; +// +// The generated config is served from /v1/opamp, which agents reach without +// authenticating, so this URL is readable by anyone who can POST there. A URL +// carrying `user:token@` — the usual way to point at a hosted Prometheus — +// would hand that credential out with it, so such an endpoint is rejected +// rather than used. Non-http schemes go the same way: the exporter only speaks +// HTTP, so anything else is a misconfiguration. +export const SPAN_METRICS_PROM_RW_ENDPOINT = (() => { + const raw = env.SPAN_METRICS_PROM_RW_ENDPOINT; + if (!raw) return undefined; + let url: URL; + try { + url = new URL(raw); + } catch { + return undefined; + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') return undefined; + if (url.username || url.password) return undefined; + return raw; +})(); export const IS_SPAN_METRICS_PROM_RW_ENABLED = env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT; diff --git a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts index 2a2f17833d..3f46602bf2 100644 --- a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts +++ b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts @@ -47,6 +47,28 @@ import { resolve } from 'path'; import { buildOtelCollectorConfig } from '@/opamp/controllers/opampController'; +/** + * Pipeline names declared under `service.pipelines:` in the bootstrap collector + * config. Scanned rather than parsed because no YAML library is a dependency of + * this package and the block is a handful of fixed-indent keys. + */ +const bootstrapPipelineNames = (yamlText: string): string[] => { + const lines = yamlText.split('\n'); + const start = lines.indexOf(' pipelines:'); + if (start === -1) return []; + const names: string[] = []; + for (const line of lines.slice(start + 1)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const indent = line.length - line.trimStart().length; + if (indent <= 2) break; // dedented out of the pipelines block + if (indent !== 4) continue; // a key within a pipeline, e.g. `processors:` + const match = /^([^:\s]+):$/.exec(trimmed); + if (match) names.push(match[1]); + } + return names; +}; + const resetConfig = () => { configState.IS_ALL_IN_ONE_IMAGE = false; configState.IS_LOCAL_APP_MODE = false; @@ -223,38 +245,45 @@ describe('opampController', () => { // `/` ids share the base type, which is what gets registered. const baseType = (id: string) => id.split('/')[0]; // gomod lines end in `` — e.g. `.../connector/spanmetricsconnector` - // for type `spanmetrics`. Matching the suffixed forms is what makes - // `span_metrics` (the bug this test exists for) fail to resolve. - const registers = (id: string) => - ['connector', 'exporter', 'receiver', 'processor'].some(kind => - builderConfig.includes(`/${baseType(id)}${kind}`), - ); + // for type `spanmetrics`. The kind has to match the section the id was + // declared in, not just any kind: `prometheus` is a real receiver, so an + // OR across kinds would wave through an `exporters.prometheus` entry that + // the build has no exporter for. + const registers = (id: string, kind: string) => + builderConfig.includes(`/${baseType(id)}${kind}`); + + // The discrimination the loop below relies on: the build has a Prometheus + // receiver but no Prometheus exporter. + expect(registers('prometheus', 'receiver')).toBe(true); + expect(registers('prometheus', 'exporter')).toBe(false); // Core components shipped in the collector binary rather than declared as // contrib gomods in builder-config.yaml. const CORE_COMPONENTS = ['nop', 'debug', 'memory_limiter', 'batch']; - const declaredIds = [ - ...Object.keys(cfg.connectors ?? {}), - ...Object.keys(cfg.exporters ?? {}), - ...Object.keys(cfg.receivers ?? {}), + const declaredIds: Array<[string, string]> = [ + ...Object.keys(cfg.connectors ?? {}).map( + id => [id, 'connector'] as [string, string], + ), + ...Object.keys(cfg.exporters ?? {}).map( + id => [id, 'exporter'] as [string, string], + ), + ...Object.keys(cfg.receivers ?? {}).map( + id => [id, 'receiver'] as [string, string], + ), ]; // Assert we actually looked at something — an empty set would make every // loop below vacuously green, which is how a bad type name shipped. expect(declaredIds.length).toBeGreaterThan(0); - for (const id of declaredIds) { + for (const [id, kind] of declaredIds) { if (CORE_COMPONENTS.includes(baseType(id))) continue; - expect(registers(id)).toBe(true); + expect([id, registers(id, kind)]).toEqual([id, true]); } - // Every pipeline reference must resolve to a declared component. Processors - // are included: they're named here but declared in - // docker/otel-collector/config.yaml, so they can't be checked against the - // declared set — but they still have to be types the build registers, since - // an unregistered processor fails the whole config to decode just like an - // unregistered connector. - const declared = new Set(declaredIds); + // Every pipeline reference must also resolve to a component this config + // declares — a pipeline naming an undeclared id fails to decode too. + const declared = new Set(declaredIds.map(([id]) => id)); let pipelineRefs = 0; for (const pipeline of Object.values(cfg.service.pipelines)) { for (const id of [ @@ -264,13 +293,109 @@ describe('opampController', () => { expect(declared).toContain(id); pipelineRefs++; } - for (const id of pipeline?.processors ?? []) { - if (CORE_COMPONENTS.includes(baseType(id))) continue; - expect(registers(id)).toBe(true); - pipelineRefs++; - } } expect(pipelineRefs).toBeGreaterThan(0); }); + + // The supervisor merges the bootstrap config.yaml with the remote config + // unconditionally, and the collector rejects a pipeline with no receivers or + // no exporters ("must have at least one receiver") — failing the whole + // config, not just that pipeline. So any pipeline key the bootstrap declares + // has to be one the generated config always fills in. Declaring a + // flag-gated pipeline there leaves the collector unable to start whenever + // the flag is off, which is exactly the bug this pins. + it.each([ + ['off', false], + ['on', true], + ])( + 'fills in every bootstrap-declared pipeline with span metrics %s', + (_label, enabled) => { + configState.IS_SPAN_METRICS_ENABLED = enabled; + + const bootstrapPipelines = bootstrapPipelineNames( + readFileSync( + resolve( + __dirname, + '../../../../../../docker/otel-collector/config.yaml', + ), + 'utf8', + ), + ); + // An empty scan would make the loop below vacuously green. + expect(bootstrapPipelines.length).toBeGreaterThan(0); + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + for (const name of bootstrapPipelines) { + const pipeline = cfg.service.pipelines[name]; + expect([name, pipeline?.receivers?.length ?? 0]).not.toEqual([ + name, + 0, + ]); + expect([name, pipeline?.exporters?.length ?? 0]).not.toEqual([ + name, + 0, + ]); + } + }, + ); + + it('keeps the derived series bounded', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + // Temporality is cumulative, so nothing evicts a series once created. + // Without a limit, one caller-supplied dimension (a tenant id, a raw + // unparameterised path) grows collector memory and the ClickHouse write + // volume without bound. + expect( + cfg.connectors?.spanmetrics?.aggregation_cardinality_limit, + ).toBeGreaterThan(0); + // The limit is per resource-cache entry, so it bounds nothing unless the + // resource key is bounded too. Left at its default the key is every + // resource attribute, and per-pod attributes would multiply the ceiling by + // the cache size. + expect( + cfg.connectors?.spanmetrics?.resource_metrics_key_attributes, + ).toEqual([ + 'service.name', + 'service.namespace', + 'deployment.environment.name', + ]); + const dimensions = cfg.connectors?.spanmetrics?.dimensions.map( + d => d.name, + ); + expect(dimensions).toEqual([ + 'http.route', + 'http.request.method', + 'http.response.status_code', + ]); + }); + + it('omits the remote-write exporter when the endpoint is unset', () => { + // config.ts derives IS_SPAN_METRICS_PROM_RW_ENABLED from the endpoint + // being present, but the controller must not depend on that: an exporter + // with no `endpoint` fails the whole config to decode, taking ingestion + // down rather than just skipping this sink. + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = undefined; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + expect( + cfg.exporters?.['prometheusremotewrite/spanmetrics'], + ).toBeUndefined(); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + ]); + }); }); }); diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 67a9b77c5f..638ca9b0e6 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -46,6 +46,13 @@ const opampRemoteConfigsCounter = getCounter('hyperdx.opamp.remote_configs', { 'Count of OpAMP remote collector configs sent back to agents in a ServerToAgent response.', }); +type PrometheusRemoteWriteExporter = { + endpoint: string; + resource_to_telemetry_conversion: { + enabled: boolean; + }; +}; + type CollectorConfig = { extensions: Record; receivers: { @@ -95,7 +102,9 @@ type CollectorConfig = { }; }; }; - connectors?: { + // Always constructed below, so the blocks that extend them can index + // without narrowing. + connectors: { 'routing/logs'?: { default_pipelines: string[]; error_mode: string; @@ -110,10 +119,11 @@ type CollectorConfig = { dimensions: Array<{ name: string }>; exemplars: { enabled: boolean }; metrics_flush_interval: string; - namespace?: string; + resource_metrics_key_attributes: string[]; + aggregation_cardinality_limit: number; }; }; - exporters?: { + exporters: { nop?: null; debug?: { verbosity: string; @@ -153,24 +163,14 @@ type CollectorConfig = { max_elapsed_time: string; }; }; - prometheusremotewrite?: { - endpoint: string; - tls: { - insecure: boolean; - }; - resource_to_telemetry_conversion: { - enabled: boolean; - }; - }; - 'prometheusremotewrite/spanmetrics'?: { - endpoint: string; - tls: { - insecure: boolean; - }; - resource_to_telemetry_conversion: { - enabled: boolean; - }; + prometheusremotewrite?: PrometheusRemoteWriteExporter & { + tls: { insecure: boolean }; }; + // No `tls` block. On an HTTP exporter the URL scheme decides whether TLS is + // used and verification is on by default; `insecure` only means anything to + // gRPC transports, so it is inert on the exporter above rather than + // weakening it. Omitted here so nobody reads it as a knob that does. + 'prometheusremotewrite/spanmetrics'?: PrometheusRemoteWriteExporter; }; service: { extensions: string[]; @@ -344,7 +344,7 @@ export const buildOtelCollectorConfig = ( 'otlp/hyperdx', ); - if (config.IS_PROMQL_ENABLED && otelCollectorConfig.exporters) { + if (config.IS_PROMQL_ENABLED) { otelCollectorConfig.exporters.prometheusremotewrite = { endpoint: 'http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write', tls: { @@ -361,11 +361,7 @@ export const buildOtelCollectorConfig = ( }; } - if ( - config.IS_SPAN_METRICS_ENABLED && - otelCollectorConfig.connectors && - otelCollectorConfig.exporters - ) { + if (config.IS_SPAN_METRICS_ENABLED) { // Derive request metrics (with trace exemplars) from spans. The connector // consumes the traces pipeline and feeds a dedicated metrics pipeline, so // the resulting `traces.span.metrics.*` land in ClickHouse with @@ -393,16 +389,50 @@ export const buildOtelCollectorConfig = ( // observed range, giving sub-percent quantile error at any latency. exponential: { max_size: 160 }, }, + // Stable HTTP semconv names — `http.method`/`http.status_code` are the + // pre-1.23 spellings and are absent from current SDKs. Every dimension + // here is bounded by the application's own route table or by the HTTP + // spec; free-form or caller-supplied attributes (a tenant id, a raw + // unparameterised path) are deliberately excluded, because each distinct + // combination is a separate series held in collector memory and written + // to ClickHouse on every flush. + // + // The connector always adds `service.name`, `span.name`, `span.kind` and + // `status.code` on top of these, and they cannot be turned off. Of those + // `span.name` is the one the instrumentation controls, so an SDK that + // puts raw paths or ids in span names widens the label set regardless of + // what is listed here. dimensions: [ { name: 'http.route' }, - { name: 'http.method' }, - { name: 'host.region' }, - { name: 'app.tenant_id' }, - { name: 'http.status_code' }, + { name: 'http.request.method' }, + { name: 'http.response.status_code' }, ], exemplars: { enabled: true }, metrics_flush_interval: '15s', + // Backstop for the above: temporality is cumulative, so series are never + // evicted on their own. Past this many, the connector folds further + // combinations into an overflow series rather than growing without + // bound. + // + // The limit applies per entry in the resource cache, not globally, so it + // only bounds anything if the resource key is itself bounded. By default + // that key is every resource attribute — including per-pod and + // per-process ones — which would make the real ceiling + // resource_metrics_cache_size (1000) x this limit. Keying on the service + // instead keeps one aggregation per service, and stops counters resetting + // when a pod restarts and its resource key changes. + resource_metrics_key_attributes: [ + 'service.name', + 'service.namespace', + 'deployment.environment.name', + ], + aggregation_cardinality_limit: 10_000, }; + // A connector runs inline in the pipeline fan-out rather than behind + // exporterhelper's queue and retry, so unlike the clickhouse exporter + // beside it there is nothing decoupling it from trace ingestion — time + // spent aggregating is time the traces pipeline waits. That is inherent to + // connectors; it is the reason the dimensions above are kept small. otelCollectorConfig.service.pipelines.traces.exporters.push( 'spanmetrics', ); @@ -411,15 +441,33 @@ export const buildOtelCollectorConfig = ( // Optionally also remote-write the derived metrics (with exemplars) to a // Prometheus endpoint, so the native Prometheus `query_exemplars` path can // be exercised against the same real, generated data. - if (config.IS_SPAN_METRICS_PROM_RW_ENABLED) { + // + // Keyed off the endpoint rather than the flag alone: an exporter with no + // `endpoint` fails the whole config to decode, which would take ingestion + // down (see the connector comment above) over an optional extra sink. + const spanMetricsRemoteWriteEndpoint = + config.IS_SPAN_METRICS_PROM_RW_ENABLED + ? config.SPAN_METRICS_PROM_RW_ENDPOINT + : undefined; + if (spanMetricsRemoteWriteEndpoint) { otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { - // Guaranteed set by IS_SPAN_METRICS_PROM_RW_ENABLED above. - endpoint: config.SPAN_METRICS_PROM_RW_ENDPOINT!, - tls: { insecure: true }, - resource_to_telemetry_conversion: { enabled: true }, + endpoint: spanMetricsRemoteWriteEndpoint, + // Off, unlike the promql exporter above: that one writes to our own + // ClickHouse-backed Prometheus, whereas this endpoint is a third party. + // Promoting every resource attribute to a label would egress host, pod + // and namespace there, and it happens in the exporter — after, and so + // unbounded by, the connector's aggregation_cardinality_limit. + resource_to_telemetry_conversion: { enabled: false }, }; spanMetricsExporters.push('prometheusremotewrite/spanmetrics'); } + // Sets `processors:` even though the comment on `pipelines` above says the + // remote config should not, which costs CUSTOM_OTELCOL_CONFIG_FILE + // overrides for this one pipeline (as `metrics/promql` already does). The + // alternative — declaring them in the bootstrap config — is worse: this + // pipeline only exists when the flag is on, so the bootstrap would be left + // holding a pipeline with no receivers and no exporters, which fails + // collector validation outright and stops the agent starting at all. otelCollectorConfig.service.pipelines['metrics/spanmetrics'] = { receivers: ['spanmetrics'], processors: ['memory_limiter', 'batch'], diff --git a/packages/otel-collector/README.md b/packages/otel-collector/README.md index 95cf13a557..1b55b488b4 100644 --- a/packages/otel-collector/README.md +++ b/packages/otel-collector/README.md @@ -92,20 +92,22 @@ custom OTel configurations without rebuilding the collector. ### Exporters -| Component | Module | Used in | -| ------------ | ------- | ------------------------------------ | -| `clickhouse` | contrib | standalone configs, OpAMP controller | -| `debug` | core | OpAMP controller | -| `nop` | core | OpAMP controller | -| `otlp` | core | included for utility | -| `otlphttp` | core | custom.config.yaml | +| Component | Module | Used in | +| ----------------------- | ------- | ----------------------------------------------------------------------------- | +| `clickhouse` | contrib | standalone configs, OpAMP controller | +| `debug` | core | OpAMP controller | +| `nop` | core | OpAMP controller | +| `otlp` | core | included for utility | +| `otlphttp` | core | custom.config.yaml | +| `prometheusremotewrite` | contrib | OpAMP controller (opt-in via `ENABLE_PROMQL` / `ENABLE_SPAN_METRICS_PROM_RW`) | ### Connectors -| Component | Module | Used in | -| --------- | ------- | ------------------------------------ | -| `forward` | core | included for utility | -| `routing` | contrib | standalone configs, OpAMP controller | +| Component | Module | Used in | +| ------------- | ------- | ----------------------------------------------- | +| `forward` | core | included for utility | +| `routing` | contrib | standalone configs, OpAMP controller | +| `spanmetrics` | contrib | OpAMP controller (behind `ENABLE_SPAN_METRICS`) | ### Extensions @@ -131,17 +133,16 @@ custom OTel configurations without rebuilding the collector. ## Ingesting Datadog traces, metrics, and logs -The `datadogreceiver` contrib component is compiled into the binary so a -Datadog Agent can ship **traces, metrics, and logs** to HyperDX. The receiver -runs a single HTTP server on `:8126` that serves the Datadog intake API for -all three signals and translates them into OTLP, which flows through the -existing `traces`, `metrics`, and `logs` pipelines into ClickHouse. It is -**opt-in**: - -- In OpAMP supervisor mode, set `ENABLE_DATADOG_RECEIVER=true` on the API/ - OpAMP process. `buildOtelCollectorConfig()` then emits the `datadog` - receiver (listening on `0.0.0.0:8126`) and attaches it to the `traces`, - `metrics`, and `logs/in` pipelines. +The `datadogreceiver` contrib component is compiled into the binary so a Datadog +Agent can ship **traces, metrics, and logs** to HyperDX. The receiver runs a +single HTTP server on `:8126` that serves the Datadog intake API for all three +signals and translates them into OTLP, which flows through the existing +`traces`, `metrics`, and `logs` pipelines into ClickHouse. It is **opt-in**: + +- In OpAMP supervisor mode, set `ENABLE_DATADOG_RECEIVER=true` on the API/ OpAMP + process. `buildOtelCollectorConfig()` then emits the `datadog` receiver + (listening on `0.0.0.0:8126`) and attaches it to the `traces`, `metrics`, and + `logs/in` pipelines. - In standalone mode, add a `datadog` receiver block to your collector config and add `datadog` to the `traces`, `metrics`, and `logs/in` pipeline receivers. @@ -152,29 +153,75 @@ config for logs). ### Authentication -In OpAMP supervisor mode, when collector authentication is enforced the -receiver authenticates against the team API keys (same as `otlp/hyperdx`) via -the `DD-API-KEY` header. Set the Datadog Agent's `DD_API_KEY` to a HyperDX -team ingestion API key. +In OpAMP supervisor mode, when collector authentication is enforced the receiver +authenticates against the team API keys (same as `otlp/hyperdx`) via the +`DD-API-KEY` header. Set the Datadog Agent's `DD_API_KEY` to a HyperDX team +ingestion API key. + +## Deriving request metrics from spans + +The `spanmetricsconnector` contrib component is compiled into the binary so the +collector can derive RED-style request metrics from the spans already flowing +through the traces pipeline. Each metric carries **exemplars** — trace and span +ids pointing back at the individual requests it was measured from — so a spike +on a latency chart links straight to a slow trace. + +Set `ENABLE_SPAN_METRICS=true` on the **API** (not the collector: the OpAMP +controller generates the collector's config). Off by default. + +**Roll the collector image out first.** A config naming a component type the +binary does not register fails to decode as a whole, and the bootstrap +`config.yaml` supplies no receivers or exporters of its own — so enabling this +against a collector built before `spanmetricsconnector` was added leaves it with +no working pipelines at all, rather than just without span metrics. + +### Bounding the series count + +Temporality is cumulative, so a series is never evicted once created. Two knobs +keep that bounded, and both matter: + +- `dimensions` is limited to `http.route`, `http.request.method` and + `http.response.status_code`. Each is bounded by the application's own route + table or by the HTTP spec. The connector also always adds `service.name`, + `span.name`, `span.kind` and `status.code`, which cannot be turned off — + `span.name` is the one the instrumentation controls, so an SDK that puts raw + paths or ids in span names widens the label set regardless. +- `aggregation_cardinality_limit` caps combinations **per resource-cache + entry**, not globally. That only bounds anything because + `resource_metrics_key_attributes` narrows the cache key to the service; left + at its default the key is every resource attribute, including per-pod ones, + and the real ceiling becomes the cache size multiplied by the limit. + +### Remote-writing to Prometheus + +`ENABLE_SPAN_METRICS_PROM_RW=true` plus `SPAN_METRICS_PROM_RW_ENDPOINT=` +additionally ships the derived metrics to a Prometheus endpoint, so Prometheus's +native `/api/v1/query_exemplars` path can be exercised against real data. + +The generated config is served from the unauthenticated OpAMP endpoint, so the +URL is readable by anyone who can reach it. An endpoint carrying `user:token@` +credentials — the usual way to point at a hosted Prometheus — is therefore +rejected rather than inlined, as is any non-HTTP scheme. Resource attributes are +not promoted to labels on this exporter, so host, pod and namespace are not sent +to a third party. ## Overriding base components via `CUSTOM_OTELCOL_CONFIG_FILE` -The collector ships with a default `memory_limiter` processor sized for a -small (~2 GiB) container. On larger pods you typically want to switch to -`limit_percentage`/`spike_limit_percentage` mode so the limit scales with -the pod's memory allocation. +The collector ships with a default `memory_limiter` processor sized for a small +(~2 GiB) container. On larger pods you typically want to switch to +`limit_percentage`/`spike_limit_percentage` mode so the limit scales with the +pod's memory allocation. The OTel `confmap` package merges YAML maps **leaf-by-leaf** rather than -replacing a block wholesale, and the `memorylimiterprocessor` silently -prefers `limit_mib` over `limit_percentage` when both are set. The -combination means you cannot switch the default `memory_limiter` to -percentage mode by leaf-merging into the existing block — your percentage -values land in `effective.yaml` but the inherited mib values still win at -runtime. +replacing a block wholesale, and the `memorylimiterprocessor` silently prefers +`limit_mib` over `limit_percentage` when both are set. The combination means you +cannot switch the default `memory_limiter` to percentage mode by leaf-merging +into the existing block — your percentage values land in `effective.yaml` but +the inherited mib values still win at runtime. -The supported pattern is to **define a new processor with a different -name** and swap the pipeline `processors:` lists in -`CUSTOM_OTELCOL_CONFIG_FILE` to reference it: +The supported pattern is to **define a new processor with a different name** and +swap the pipeline `processors:` lists in `CUSTOM_OTELCOL_CONFIG_FILE` to +reference it: ```yaml # custom.config.yaml @@ -196,18 +243,18 @@ service: processors: [memory_limiter/custom, batch] ``` -After restart, the collector instantiates `memory_limiter/custom` (and not -the unused default `memory_limiter`). You can confirm by checking -`/etc/otel/supervisor-data/effective.yaml` and the `"Memory limiter -configured"` log line emitted at collector start. +After restart, the collector instantiates `memory_limiter/custom` (and not the +unused default `memory_limiter`). You can confirm by checking +`/etc/otel/supervisor-data/effective.yaml` and the `"Memory limiter configured"` +log line emitted at collector start. ### Example: tuning the `batch` processor -The default `batch` processor is sized for ClickHouse -(`send_batch_size: 10000`, `timeout: 5s`). If your workload needs -lower-latency exports — for example, latency-sensitive traces or a -short-window smoke test that asserts shortly after emitting — define a -new `batch` with the tuning you want and swap the pipelines to use it: +The default `batch` processor is sized for ClickHouse (`send_batch_size: 10000`, +`timeout: 5s`). If your workload needs lower-latency exports — for example, +latency-sensitive traces or a short-window smoke test that asserts shortly after +emitting — define a new `batch` with the tuning you want and swap the pipelines +to use it: ```yaml # custom.config.yaml @@ -227,23 +274,21 @@ service: Notes: -- You only need to re-declare the pipelines you want to retune; pipelines - you don't mention keep the default `batch` from the base config. In the - example above, `metrics` and `logs/out-rrweb` continue using the default - `batch`. +- You only need to re-declare the pipelines you want to retune; pipelines you + don't mention keep the default `batch` from the base config. In the example + above, `metrics` and `logs/out-rrweb` continue using the default `batch`. - Processor list order matters. Keep `memory_limiter` ahead of the batch processor so back-pressure applies before batching. -- Larger batches are friendlier to ClickHouse (fewer inserts, fewer - MergeTree parts). The defaults — `send_batch_size: 10000`, `timeout: 5s` - — are the recommended starting point; only tune them when you have a - specific reason. +- Larger batches are friendlier to ClickHouse (fewer inserts, fewer MergeTree + parts). The defaults — `send_batch_size: 10000`, `timeout: 5s` — are the + recommended starting point; only tune them when you have a specific reason. - You can combine swap blocks. Define both `memory_limiter/custom` and - `batch/lowlatency`, and reference both in the same pipeline - (e.g. `processors: [memory_limiter/custom, batch/lowlatency]`). + `batch/lowlatency`, and reference both in the same pipeline (e.g. + `processors: [memory_limiter/custom, batch/lowlatency]`). You can verify the new processor is actually running (not just present in -`effective.yaml`) by querying the collector's Prometheus metrics endpoint -on port `8888` and looking for the new `processor` label: +`effective.yaml`) by querying the collector's Prometheus metrics endpoint on +port `8888` and looking for the new `processor` label: ```sh curl -s http://localhost:8888/metrics | grep 'processor="batch/lowlatency"' @@ -253,32 +298,30 @@ curl -s http://localhost:8888/metrics | grep 'processor="batch/lowlatency"' ### Lighter-weight option: env vars for the default `batch` -If you only need to tune `send_batch_size`, `send_batch_max_size`, or -`timeout` on the default `batch` processor and don't need to define a -second processor, the existing env vars also work without a custom config -file: +If you only need to tune `send_batch_size`, `send_batch_max_size`, or `timeout` +on the default `batch` processor and don't need to define a second processor, +the existing env vars also work without a custom config file: - `HYPERDX_OTEL_BATCH_SEND_BATCH_SIZE` (default `10000`) -- `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE` (default `0`, meaning no upper - bound) +- `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE` (default `0`, meaning no upper bound) - `HYPERDX_OTEL_BATCH_TIMEOUT` (default `5s`) -These are read directly from the base `config.yaml`, so they apply -everywhere the default `batch` is referenced. Use the swap pattern when -you need different settings for different pipelines, when you want to -combine batch + memory_limiter changes, or when you want to override -`memory_limiter` (which has no equivalent env-var path). +These are read directly from the base `config.yaml`, so they apply everywhere +the default `batch` is referenced. Use the swap pattern when you need different +settings for different pipelines, when you want to combine batch + +memory_limiter changes, or when you want to override `memory_limiter` (which has +no equivalent env-var path). The same swap pattern works for any other base processor (`transform`, -`resourcedetection`, …) — define a new component with a different name -and re-declare the pipelines that should use it. - -> Pipeline `processors:` lists live in `docker/otel-collector/config.yaml` -> (for OpAMP supervisor mode) and `docker/otel-collector/config.standalone.yaml` -> (for standalone mode). The OpAMP remote config from -> `packages/api/src/opamp/controllers/opampController.ts` intentionally -> does **not** set `processors:` on pipelines, so your bootstrap+custom -> merge is not overwritten. +`resourcedetection`, …) — define a new component with a different name and +re-declare the pipelines that should use it. + +> Pipeline `processors:` lists live in `docker/otel-collector/config.yaml` (for +> OpAMP supervisor mode) and `docker/otel-collector/config.standalone.yaml` (for +> standalone mode). The OpAMP remote config from +> `packages/api/src/opamp/controllers/opampController.ts` intentionally does +> **not** set `processors:` on pipelines, so your bootstrap+custom merge is not +> overwritten. ## Upgrading the OTel Collector version