diff --git a/.changeset/spanmetrics-connector.md b/.changeset/spanmetrics-connector.md new file mode 100644 index 0000000000..685cca2d7f --- /dev/null +++ b/.changeset/spanmetrics-connector.md @@ -0,0 +1,37 @@ +--- +'@hyperdx/otel-collector': minor +'@hyperdx/api': minor +--- + +feat: derive request metrics with trace exemplars from spans + +Adds the OpenTelemetry `spanmetricsconnector` to the collector build and wires +it into the generated OpAMP config behind `ENABLE_SPAN_METRICS`, off by default. +The connector reads the traces pipeline and feeds a dedicated metrics pipeline, +so `traces.span.metrics.*` land in ClickHouse with `Exemplars.*` pointing back +at the spans they were measured from. + +Histogram buckets are exponential rather than a fixed ladder: an explicit ladder +puts everything slow into one wide top bucket, so a high quantile interpolates +well past the slowest real request and no exemplar can sit on the plotted line. + +Dimensions are limited to `http.route`, `http.request.method` and +`http.response.status_code` — each bounded by the application's route table or +by the HTTP spec — and the connector is given an explicit +`aggregation_cardinality_limit`. Temporality is cumulative, so nothing evicts a +series once created; one free-form dimension would grow collector memory and the +ClickHouse write volume without bound. + +`ENABLE_SPAN_METRICS_PROM_RW` additionally remote-writes the derived metrics to +a Prometheus endpoint, for exercising Prometheus's native exemplar path. Because +the generated config is served from the unauthenticated OpAMP endpoint, an +endpoint URL carrying `user:token@` credentials is rejected rather than inlined, +as are non-HTTP schemes. Resource attributes are not promoted to labels on this +exporter: that happens after the connector's cardinality limit and would send +host, pod and namespace to a third party. + +Requires a collector built from the current `builder-config.yaml`. Roll the +collector image out before enabling the flag: a config naming a component type +the binary does not register fails to decode as a whole, and the shipped +`docker/otel-collector/config.yaml` defines no pipelines of its own, so a +rejected remote config leaves the collector with nothing to run. diff --git a/.gitignore b/.gitignore index 31c6b70198..1f3ae85a33 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,7 @@ docker-compose.prod.yml # webstorm .idea + +# Stryker mutation-testing output +packages/*/reports/ +packages/*/.stryker-tmp/ diff --git a/packages/api/src/__tests__/config.test.ts b/packages/api/src/__tests__/config.test.ts index fedf486e0d..40c3ffcdae 100644 --- a/packages/api/src/__tests__/config.test.ts +++ b/packages/api/src/__tests__/config.test.ts @@ -59,4 +59,63 @@ describe('config', () => { }); }); }); + + describe('SPAN_METRICS_PROM_RW_ENDPOINT', () => { + const ORIGINAL = process.env.SPAN_METRICS_PROM_RW_ENDPOINT; + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.SPAN_METRICS_PROM_RW_ENDPOINT; + } else { + process.env.SPAN_METRICS_PROM_RW_ENDPOINT = ORIGINAL; + } + jest.resetModules(); + }); + + const resolve = (raw: string | undefined) => { + if (raw === undefined) { + delete process.env.SPAN_METRICS_PROM_RW_ENDPOINT; + } else { + process.env.SPAN_METRICS_PROM_RW_ENDPOINT = raw; + } + let resolved: string | undefined; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports, n/no-missing-require + resolved = require('@/config').SPAN_METRICS_PROM_RW_ENDPOINT; + }); + return resolved; + }; + + it('accepts a plain http or https endpoint', () => { + expect(resolve('http://prometheus:9090/api/v1/write')).toBe( + 'http://prometheus:9090/api/v1/write', + ); + expect(resolve('https://prom.example.com/api/v1/push')).toBe( + 'https://prom.example.com/api/v1/push', + ); + }); + + // The generated collector config is served from the unauthenticated OpAMP + // endpoint, so a credential in this URL is readable by anyone who can POST + // there. Pointing at a hosted Prometheus this way is the common case, which + // is why it is rejected rather than passed along. + it('rejects an endpoint carrying credentials', () => { + expect(resolve('https://user:token@prom.example.com/api/v1/push')).toBe( + undefined, + ); + expect(resolve('https://token@prom.example.com/api/v1/push')).toBe( + undefined, + ); + }); + + it('rejects a non-HTTP scheme or an unparseable URL', () => { + expect(resolve('file:///etc/passwd')).toBe(undefined); + expect(resolve('prometheus:9090')).toBe(undefined); + expect(resolve('not a url')).toBe(undefined); + }); + + it('is undefined when unset', () => { + expect(resolve(undefined)).toBe(undefined); + }); + }); }); diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 7c6a242969..85023e30ce 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -57,6 +57,46 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; +// Opt-in: have the collector derive request metrics (with trace exemplars) from +// spans via the spanmetrics connector. Off everywhere by default — set +// ENABLE_SPAN_METRICS=true to have the telemetry-generator's traces produce +// coherent metric exemplars end-to-end. +// +// Requires a collector built from the current builder-config.yaml. An older +// image does not register the `spanmetrics` type, and a config naming an +// unregistered type fails to decode as a whole, so the agent would fall back to +// no pipelines at all. Roll the collector image out before setting this. +export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; + +// Opt-in: also remote-write the span-derived metrics (with exemplars) to a +// Prometheus endpoint so the native Prometheus query_exemplars path can be +// tested against real data. The endpoint is resolved here (API side) and +// inlined into the generated collector config, so the collector container does +// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the +// endpoint to be set; without it the feature stays disabled. +// +// The generated config is served from /v1/opamp, which agents reach without +// authenticating, so this URL is readable by anyone who can POST there. A URL +// carrying `user:token@` — the usual way to point at a hosted Prometheus — +// would hand that credential out with it, so such an endpoint is rejected +// rather than used. Non-http schemes go the same way: the exporter only speaks +// HTTP, so anything else is a misconfiguration. +export const SPAN_METRICS_PROM_RW_ENDPOINT = (() => { + const raw = env.SPAN_METRICS_PROM_RW_ENDPOINT; + if (!raw) return undefined; + let url: URL; + try { + url = new URL(raw); + } catch { + return undefined; + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') return undefined; + if (url.username || url.password) return undefined; + return raw; +})(); +export const IS_SPAN_METRICS_PROM_RW_ENABLED = + env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT; + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts index f86baa8039..3f46602bf2 100644 --- a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts +++ b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts @@ -7,6 +7,9 @@ const configState = { INGESTION_API_KEY: '' as string, IS_PROMQL_ENABLED: false, ENABLE_DATADOG_RECEIVER: false, + IS_SPAN_METRICS_ENABLED: false, + IS_SPAN_METRICS_PROM_RW_ENABLED: false, + SPAN_METRICS_PROM_RW_ENDPOINT: undefined as string | undefined, }; jest.mock('@/config', () => ({ @@ -28,10 +31,44 @@ jest.mock('@/config', () => ({ get ENABLE_DATADOG_RECEIVER() { return configState.ENABLE_DATADOG_RECEIVER; }, + get IS_SPAN_METRICS_ENABLED() { + return configState.IS_SPAN_METRICS_ENABLED; + }, + get IS_SPAN_METRICS_PROM_RW_ENABLED() { + return configState.IS_SPAN_METRICS_PROM_RW_ENABLED; + }, + get SPAN_METRICS_PROM_RW_ENDPOINT() { + return configState.SPAN_METRICS_PROM_RW_ENDPOINT; + }, })); +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + import { buildOtelCollectorConfig } from '@/opamp/controllers/opampController'; +/** + * Pipeline names declared under `service.pipelines:` in the bootstrap collector + * config. Scanned rather than parsed because no YAML library is a dependency of + * this package and the block is a handful of fixed-indent keys. + */ +const bootstrapPipelineNames = (yamlText: string): string[] => { + const lines = yamlText.split('\n'); + const start = lines.indexOf(' pipelines:'); + if (start === -1) return []; + const names: string[] = []; + for (const line of lines.slice(start + 1)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const indent = line.length - line.trimStart().length; + if (indent <= 2) break; // dedented out of the pipelines block + if (indent !== 4) continue; // a key within a pipeline, e.g. `processors:` + const match = /^([^:\s]+):$/.exec(trimmed); + if (match) names.push(match[1]); + } + return names; +}; + const resetConfig = () => { configState.IS_ALL_IN_ONE_IMAGE = false; configState.IS_LOCAL_APP_MODE = false; @@ -39,6 +76,9 @@ const resetConfig = () => { configState.INGESTION_API_KEY = ''; configState.IS_PROMQL_ENABLED = false; configState.ENABLE_DATADOG_RECEIVER = false; + configState.IS_SPAN_METRICS_ENABLED = false; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = false; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = undefined; }; describe('opampController', () => { @@ -123,4 +163,239 @@ describe('opampController', () => { expect(cfg.service.extensions).toContain('bearertokenauth/hyperdx'); }); }); + + describe('buildOtelCollectorConfig span metrics', () => { + it('omits the span metrics connector when the flag is off (default)', () => { + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + expect(cfg.connectors?.spanmetrics).toBeUndefined(); + expect(cfg.service.pipelines['metrics/spanmetrics']).toBeUndefined(); + expect(cfg.service.pipelines.traces.exporters).not.toContain( + 'spanmetrics', + ); + }); + + it('derives span metrics as exponential histograms with exemplars', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + // Exponential rather than explicit buckets: a fixed ladder's wide top + // bucket makes high quantiles interpolate well past the slowest real + // request, so no exemplar can ever sit on the plotted line. + expect(cfg.connectors?.spanmetrics?.histogram).toEqual({ + unit: 'ms', + exponential: { max_size: 160 }, + }); + expect(cfg.connectors?.spanmetrics?.exemplars).toEqual({ + enabled: true, + }); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + ]); + // The connector has to be wired at both ends or it silently does nothing. + expect(cfg.service.pipelines.traces.exporters).toContain('spanmetrics'); + expect(cfg.service.pipelines['metrics/spanmetrics']?.receivers).toEqual([ + 'spanmetrics', + ]); + }); + + it('adds a remote-write exporter for the derived metrics when enabled', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = + 'http://prometheus:9090/api/v1/write'; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + expect( + cfg.exporters?.['prometheusremotewrite/spanmetrics'], + ).toMatchObject({ endpoint: 'http://prometheus:9090/api/v1/write' }); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + 'prometheusremotewrite/spanmetrics', + ]); + }); + + // A config naming a component type the binary doesn't register fails to + // decode as a whole, and docker/otel-collector/config.yaml defines no + // pipelines of its own — so a bad type name here leaves the collector with + // no usable config at all rather than just disabling one feature. Pin every + // generated component id against what builder-config.yaml actually builds. + it('only references component types the collector build registers', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = + 'http://prometheus:9090/write'; + configState.IS_PROMQL_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + const builderConfig = readFileSync( + resolve(__dirname, '../../../../../otel-collector/builder-config.yaml'), + 'utf8', + ); + // `/` ids share the base type, which is what gets registered. + const baseType = (id: string) => id.split('/')[0]; + // gomod lines end in `` — e.g. `.../connector/spanmetricsconnector` + // for type `spanmetrics`. The kind has to match the section the id was + // declared in, not just any kind: `prometheus` is a real receiver, so an + // OR across kinds would wave through an `exporters.prometheus` entry that + // the build has no exporter for. + const registers = (id: string, kind: string) => + builderConfig.includes(`/${baseType(id)}${kind}`); + + // The discrimination the loop below relies on: the build has a Prometheus + // receiver but no Prometheus exporter. + expect(registers('prometheus', 'receiver')).toBe(true); + expect(registers('prometheus', 'exporter')).toBe(false); + + // Core components shipped in the collector binary rather than declared as + // contrib gomods in builder-config.yaml. + const CORE_COMPONENTS = ['nop', 'debug', 'memory_limiter', 'batch']; + + const declaredIds: Array<[string, string]> = [ + ...Object.keys(cfg.connectors ?? {}).map( + id => [id, 'connector'] as [string, string], + ), + ...Object.keys(cfg.exporters ?? {}).map( + id => [id, 'exporter'] as [string, string], + ), + ...Object.keys(cfg.receivers ?? {}).map( + id => [id, 'receiver'] as [string, string], + ), + ]; + // Assert we actually looked at something — an empty set would make every + // loop below vacuously green, which is how a bad type name shipped. + expect(declaredIds.length).toBeGreaterThan(0); + + for (const [id, kind] of declaredIds) { + if (CORE_COMPONENTS.includes(baseType(id))) continue; + expect([id, registers(id, kind)]).toEqual([id, true]); + } + + // Every pipeline reference must also resolve to a component this config + // declares — a pipeline naming an undeclared id fails to decode too. + const declared = new Set(declaredIds.map(([id]) => id)); + let pipelineRefs = 0; + for (const pipeline of Object.values(cfg.service.pipelines)) { + for (const id of [ + ...(pipeline?.receivers ?? []), + ...(pipeline?.exporters ?? []), + ]) { + expect(declared).toContain(id); + pipelineRefs++; + } + } + expect(pipelineRefs).toBeGreaterThan(0); + }); + + // The supervisor merges the bootstrap config.yaml with the remote config + // unconditionally, and the collector rejects a pipeline with no receivers or + // no exporters ("must have at least one receiver") — failing the whole + // config, not just that pipeline. So any pipeline key the bootstrap declares + // has to be one the generated config always fills in. Declaring a + // flag-gated pipeline there leaves the collector unable to start whenever + // the flag is off, which is exactly the bug this pins. + it.each([ + ['off', false], + ['on', true], + ])( + 'fills in every bootstrap-declared pipeline with span metrics %s', + (_label, enabled) => { + configState.IS_SPAN_METRICS_ENABLED = enabled; + + const bootstrapPipelines = bootstrapPipelineNames( + readFileSync( + resolve( + __dirname, + '../../../../../../docker/otel-collector/config.yaml', + ), + 'utf8', + ), + ); + // An empty scan would make the loop below vacuously green. + expect(bootstrapPipelines.length).toBeGreaterThan(0); + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + for (const name of bootstrapPipelines) { + const pipeline = cfg.service.pipelines[name]; + expect([name, pipeline?.receivers?.length ?? 0]).not.toEqual([ + name, + 0, + ]); + expect([name, pipeline?.exporters?.length ?? 0]).not.toEqual([ + name, + 0, + ]); + } + }, + ); + + it('keeps the derived series bounded', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + // Temporality is cumulative, so nothing evicts a series once created. + // Without a limit, one caller-supplied dimension (a tenant id, a raw + // unparameterised path) grows collector memory and the ClickHouse write + // volume without bound. + expect( + cfg.connectors?.spanmetrics?.aggregation_cardinality_limit, + ).toBeGreaterThan(0); + // The limit is per resource-cache entry, so it bounds nothing unless the + // resource key is bounded too. Left at its default the key is every + // resource attribute, and per-pod attributes would multiply the ceiling by + // the cache size. + expect( + cfg.connectors?.spanmetrics?.resource_metrics_key_attributes, + ).toEqual([ + 'service.name', + 'service.namespace', + 'deployment.environment.name', + ]); + const dimensions = cfg.connectors?.spanmetrics?.dimensions.map( + d => d.name, + ); + expect(dimensions).toEqual([ + 'http.route', + 'http.request.method', + 'http.response.status_code', + ]); + }); + + it('omits the remote-write exporter when the endpoint is unset', () => { + // config.ts derives IS_SPAN_METRICS_PROM_RW_ENABLED from the endpoint + // being present, but the controller must not depend on that: an exporter + // with no `endpoint` fails the whole config to decode, taking ingestion + // down rather than just skipping this sink. + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = undefined; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + expect( + cfg.exporters?.['prometheusremotewrite/spanmetrics'], + ).toBeUndefined(); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + ]); + }); + }); }); diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 6dfb5f989c..638ca9b0e6 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -46,6 +46,13 @@ const opampRemoteConfigsCounter = getCounter('hyperdx.opamp.remote_configs', { 'Count of OpAMP remote collector configs sent back to agents in a ServerToAgent response.', }); +type PrometheusRemoteWriteExporter = { + endpoint: string; + resource_to_telemetry_conversion: { + enabled: boolean; + }; +}; + type CollectorConfig = { extensions: Record; receivers: { @@ -95,7 +102,9 @@ type CollectorConfig = { }; }; }; - connectors?: { + // Always constructed below, so the blocks that extend them can index + // without narrowing. + connectors: { 'routing/logs'?: { default_pipelines: string[]; error_mode: string; @@ -105,8 +114,16 @@ type CollectorConfig = { pipelines: string[]; }>; }; + spanmetrics?: { + histogram: { unit: string; exponential: { max_size: number } }; + dimensions: Array<{ name: string }>; + exemplars: { enabled: boolean }; + metrics_flush_interval: string; + resource_metrics_key_attributes: string[]; + aggregation_cardinality_limit: number; + }; }; - exporters?: { + exporters: { nop?: null; debug?: { verbosity: string; @@ -146,15 +163,14 @@ type CollectorConfig = { max_elapsed_time: string; }; }; - prometheusremotewrite?: { - endpoint: string; - tls: { - insecure: boolean; - }; - resource_to_telemetry_conversion: { - enabled: boolean; - }; + prometheusremotewrite?: PrometheusRemoteWriteExporter & { + tls: { insecure: boolean }; }; + // No `tls` block. On an HTTP exporter the URL scheme decides whether TLS is + // used and verification is on by default; `insecure` only means anything to + // gRPC transports, so it is inert on the exporter above rather than + // weakening it. Omitted here so nobody reads it as a knob that does. + 'prometheusremotewrite/spanmetrics'?: PrometheusRemoteWriteExporter; }; service: { extensions: string[]; @@ -328,7 +344,7 @@ export const buildOtelCollectorConfig = ( 'otlp/hyperdx', ); - if (config.IS_PROMQL_ENABLED && otelCollectorConfig.exporters) { + if (config.IS_PROMQL_ENABLED) { otelCollectorConfig.exporters.prometheusremotewrite = { endpoint: 'http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write', tls: { @@ -345,6 +361,120 @@ export const buildOtelCollectorConfig = ( }; } + if (config.IS_SPAN_METRICS_ENABLED) { + // Derive request metrics (with trace exemplars) from spans. The connector + // consumes the traces pipeline and feeds a dedicated metrics pipeline, so + // the resulting `traces.span.metrics.*` land in ClickHouse with + // `Exemplars.*` pointing back at the spans they were measured from. + // + // The key MUST be `spanmetrics` — that is the component type + // spanmetricsconnector registers, and a config naming an unregistered type + // fails to decode wholesale. Because docker/otel-collector/config.yaml + // supplies no pipelines of its own, a rejected remote config leaves the + // collector with nothing to run, so a typo here takes ingestion down + // rather than just disabling this feature. + // + // Requires a collector built from the current builder-config.yaml (which + // added spanmetricsconnector). Enabling this flag against collectors built + // before that will have them reject the config for the same reason, so + // roll the collector image out first. + otelCollectorConfig.connectors.spanmetrics = { + histogram: { + unit: 'ms', + // Exponential (OTLP) / native (Prometheus) buckets rather than a fixed + // ladder. Explicit bounds put everything slow into one wide top bucket + // — with a 5s–10s bucket, a p99 interpolates towards 10s while the + // slowest real request was half that, and no exemplar can ever sit on + // the line. max_size is the connector's default; scale adapts to the + // observed range, giving sub-percent quantile error at any latency. + exponential: { max_size: 160 }, + }, + // Stable HTTP semconv names — `http.method`/`http.status_code` are the + // pre-1.23 spellings and are absent from current SDKs. Every dimension + // here is bounded by the application's own route table or by the HTTP + // spec; free-form or caller-supplied attributes (a tenant id, a raw + // unparameterised path) are deliberately excluded, because each distinct + // combination is a separate series held in collector memory and written + // to ClickHouse on every flush. + // + // The connector always adds `service.name`, `span.name`, `span.kind` and + // `status.code` on top of these, and they cannot be turned off. Of those + // `span.name` is the one the instrumentation controls, so an SDK that + // puts raw paths or ids in span names widens the label set regardless of + // what is listed here. + dimensions: [ + { name: 'http.route' }, + { name: 'http.request.method' }, + { name: 'http.response.status_code' }, + ], + exemplars: { enabled: true }, + metrics_flush_interval: '15s', + // Backstop for the above: temporality is cumulative, so series are never + // evicted on their own. Past this many, the connector folds further + // combinations into an overflow series rather than growing without + // bound. + // + // The limit applies per entry in the resource cache, not globally, so it + // only bounds anything if the resource key is itself bounded. By default + // that key is every resource attribute — including per-pod and + // per-process ones — which would make the real ceiling + // resource_metrics_cache_size (1000) x this limit. Keying on the service + // instead keeps one aggregation per service, and stops counters resetting + // when a pod restarts and its resource key changes. + resource_metrics_key_attributes: [ + 'service.name', + 'service.namespace', + 'deployment.environment.name', + ], + aggregation_cardinality_limit: 10_000, + }; + // A connector runs inline in the pipeline fan-out rather than behind + // exporterhelper's queue and retry, so unlike the clickhouse exporter + // beside it there is nothing decoupling it from trace ingestion — time + // spent aggregating is time the traces pipeline waits. That is inherent to + // connectors; it is the reason the dimensions above are kept small. + otelCollectorConfig.service.pipelines.traces.exporters.push( + 'spanmetrics', + ); + + const spanMetricsExporters = ['clickhouse']; + // Optionally also remote-write the derived metrics (with exemplars) to a + // Prometheus endpoint, so the native Prometheus `query_exemplars` path can + // be exercised against the same real, generated data. + // + // Keyed off the endpoint rather than the flag alone: an exporter with no + // `endpoint` fails the whole config to decode, which would take ingestion + // down (see the connector comment above) over an optional extra sink. + const spanMetricsRemoteWriteEndpoint = + config.IS_SPAN_METRICS_PROM_RW_ENABLED + ? config.SPAN_METRICS_PROM_RW_ENDPOINT + : undefined; + if (spanMetricsRemoteWriteEndpoint) { + otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { + endpoint: spanMetricsRemoteWriteEndpoint, + // Off, unlike the promql exporter above: that one writes to our own + // ClickHouse-backed Prometheus, whereas this endpoint is a third party. + // Promoting every resource attribute to a label would egress host, pod + // and namespace there, and it happens in the exporter — after, and so + // unbounded by, the connector's aggregation_cardinality_limit. + resource_to_telemetry_conversion: { enabled: false }, + }; + spanMetricsExporters.push('prometheusremotewrite/spanmetrics'); + } + // Sets `processors:` even though the comment on `pipelines` above says the + // remote config should not, which costs CUSTOM_OTELCOL_CONFIG_FILE + // overrides for this one pipeline (as `metrics/promql` already does). The + // alternative — declaring them in the bootstrap config — is worse: this + // pipeline only exists when the flag is on, so the bootstrap would be left + // holding a pipeline with no receivers and no exporters, which fails + // collector validation outright and stops the agent starting at all. + otelCollectorConfig.service.pipelines['metrics/spanmetrics'] = { + receivers: ['spanmetrics'], + processors: ['memory_limiter', 'batch'], + exporters: spanMetricsExporters, + }; + } + if (collectorAuthenticationEnforced) { if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) { // should never happen diff --git a/packages/otel-collector/README.md b/packages/otel-collector/README.md index 95cf13a557..1b55b488b4 100644 --- a/packages/otel-collector/README.md +++ b/packages/otel-collector/README.md @@ -92,20 +92,22 @@ custom OTel configurations without rebuilding the collector. ### Exporters -| Component | Module | Used in | -| ------------ | ------- | ------------------------------------ | -| `clickhouse` | contrib | standalone configs, OpAMP controller | -| `debug` | core | OpAMP controller | -| `nop` | core | OpAMP controller | -| `otlp` | core | included for utility | -| `otlphttp` | core | custom.config.yaml | +| Component | Module | Used in | +| ----------------------- | ------- | ----------------------------------------------------------------------------- | +| `clickhouse` | contrib | standalone configs, OpAMP controller | +| `debug` | core | OpAMP controller | +| `nop` | core | OpAMP controller | +| `otlp` | core | included for utility | +| `otlphttp` | core | custom.config.yaml | +| `prometheusremotewrite` | contrib | OpAMP controller (opt-in via `ENABLE_PROMQL` / `ENABLE_SPAN_METRICS_PROM_RW`) | ### Connectors -| Component | Module | Used in | -| --------- | ------- | ------------------------------------ | -| `forward` | core | included for utility | -| `routing` | contrib | standalone configs, OpAMP controller | +| Component | Module | Used in | +| ------------- | ------- | ----------------------------------------------- | +| `forward` | core | included for utility | +| `routing` | contrib | standalone configs, OpAMP controller | +| `spanmetrics` | contrib | OpAMP controller (behind `ENABLE_SPAN_METRICS`) | ### Extensions @@ -131,17 +133,16 @@ custom OTel configurations without rebuilding the collector. ## Ingesting Datadog traces, metrics, and logs -The `datadogreceiver` contrib component is compiled into the binary so a -Datadog Agent can ship **traces, metrics, and logs** to HyperDX. The receiver -runs a single HTTP server on `:8126` that serves the Datadog intake API for -all three signals and translates them into OTLP, which flows through the -existing `traces`, `metrics`, and `logs` pipelines into ClickHouse. It is -**opt-in**: - -- In OpAMP supervisor mode, set `ENABLE_DATADOG_RECEIVER=true` on the API/ - OpAMP process. `buildOtelCollectorConfig()` then emits the `datadog` - receiver (listening on `0.0.0.0:8126`) and attaches it to the `traces`, - `metrics`, and `logs/in` pipelines. +The `datadogreceiver` contrib component is compiled into the binary so a Datadog +Agent can ship **traces, metrics, and logs** to HyperDX. The receiver runs a +single HTTP server on `:8126` that serves the Datadog intake API for all three +signals and translates them into OTLP, which flows through the existing +`traces`, `metrics`, and `logs` pipelines into ClickHouse. It is **opt-in**: + +- In OpAMP supervisor mode, set `ENABLE_DATADOG_RECEIVER=true` on the API/ OpAMP + process. `buildOtelCollectorConfig()` then emits the `datadog` receiver + (listening on `0.0.0.0:8126`) and attaches it to the `traces`, `metrics`, and + `logs/in` pipelines. - In standalone mode, add a `datadog` receiver block to your collector config and add `datadog` to the `traces`, `metrics`, and `logs/in` pipeline receivers. @@ -152,29 +153,75 @@ config for logs). ### Authentication -In OpAMP supervisor mode, when collector authentication is enforced the -receiver authenticates against the team API keys (same as `otlp/hyperdx`) via -the `DD-API-KEY` header. Set the Datadog Agent's `DD_API_KEY` to a HyperDX -team ingestion API key. +In OpAMP supervisor mode, when collector authentication is enforced the receiver +authenticates against the team API keys (same as `otlp/hyperdx`) via the +`DD-API-KEY` header. Set the Datadog Agent's `DD_API_KEY` to a HyperDX team +ingestion API key. + +## Deriving request metrics from spans + +The `spanmetricsconnector` contrib component is compiled into the binary so the +collector can derive RED-style request metrics from the spans already flowing +through the traces pipeline. Each metric carries **exemplars** — trace and span +ids pointing back at the individual requests it was measured from — so a spike +on a latency chart links straight to a slow trace. + +Set `ENABLE_SPAN_METRICS=true` on the **API** (not the collector: the OpAMP +controller generates the collector's config). Off by default. + +**Roll the collector image out first.** A config naming a component type the +binary does not register fails to decode as a whole, and the bootstrap +`config.yaml` supplies no receivers or exporters of its own — so enabling this +against a collector built before `spanmetricsconnector` was added leaves it with +no working pipelines at all, rather than just without span metrics. + +### Bounding the series count + +Temporality is cumulative, so a series is never evicted once created. Two knobs +keep that bounded, and both matter: + +- `dimensions` is limited to `http.route`, `http.request.method` and + `http.response.status_code`. Each is bounded by the application's own route + table or by the HTTP spec. The connector also always adds `service.name`, + `span.name`, `span.kind` and `status.code`, which cannot be turned off — + `span.name` is the one the instrumentation controls, so an SDK that puts raw + paths or ids in span names widens the label set regardless. +- `aggregation_cardinality_limit` caps combinations **per resource-cache + entry**, not globally. That only bounds anything because + `resource_metrics_key_attributes` narrows the cache key to the service; left + at its default the key is every resource attribute, including per-pod ones, + and the real ceiling becomes the cache size multiplied by the limit. + +### Remote-writing to Prometheus + +`ENABLE_SPAN_METRICS_PROM_RW=true` plus `SPAN_METRICS_PROM_RW_ENDPOINT=` +additionally ships the derived metrics to a Prometheus endpoint, so Prometheus's +native `/api/v1/query_exemplars` path can be exercised against real data. + +The generated config is served from the unauthenticated OpAMP endpoint, so the +URL is readable by anyone who can reach it. An endpoint carrying `user:token@` +credentials — the usual way to point at a hosted Prometheus — is therefore +rejected rather than inlined, as is any non-HTTP scheme. Resource attributes are +not promoted to labels on this exporter, so host, pod and namespace are not sent +to a third party. ## Overriding base components via `CUSTOM_OTELCOL_CONFIG_FILE` -The collector ships with a default `memory_limiter` processor sized for a -small (~2 GiB) container. On larger pods you typically want to switch to -`limit_percentage`/`spike_limit_percentage` mode so the limit scales with -the pod's memory allocation. +The collector ships with a default `memory_limiter` processor sized for a small +(~2 GiB) container. On larger pods you typically want to switch to +`limit_percentage`/`spike_limit_percentage` mode so the limit scales with the +pod's memory allocation. The OTel `confmap` package merges YAML maps **leaf-by-leaf** rather than -replacing a block wholesale, and the `memorylimiterprocessor` silently -prefers `limit_mib` over `limit_percentage` when both are set. The -combination means you cannot switch the default `memory_limiter` to -percentage mode by leaf-merging into the existing block — your percentage -values land in `effective.yaml` but the inherited mib values still win at -runtime. +replacing a block wholesale, and the `memorylimiterprocessor` silently prefers +`limit_mib` over `limit_percentage` when both are set. The combination means you +cannot switch the default `memory_limiter` to percentage mode by leaf-merging +into the existing block — your percentage values land in `effective.yaml` but +the inherited mib values still win at runtime. -The supported pattern is to **define a new processor with a different -name** and swap the pipeline `processors:` lists in -`CUSTOM_OTELCOL_CONFIG_FILE` to reference it: +The supported pattern is to **define a new processor with a different name** and +swap the pipeline `processors:` lists in `CUSTOM_OTELCOL_CONFIG_FILE` to +reference it: ```yaml # custom.config.yaml @@ -196,18 +243,18 @@ service: processors: [memory_limiter/custom, batch] ``` -After restart, the collector instantiates `memory_limiter/custom` (and not -the unused default `memory_limiter`). You can confirm by checking -`/etc/otel/supervisor-data/effective.yaml` and the `"Memory limiter -configured"` log line emitted at collector start. +After restart, the collector instantiates `memory_limiter/custom` (and not the +unused default `memory_limiter`). You can confirm by checking +`/etc/otel/supervisor-data/effective.yaml` and the `"Memory limiter configured"` +log line emitted at collector start. ### Example: tuning the `batch` processor -The default `batch` processor is sized for ClickHouse -(`send_batch_size: 10000`, `timeout: 5s`). If your workload needs -lower-latency exports — for example, latency-sensitive traces or a -short-window smoke test that asserts shortly after emitting — define a -new `batch` with the tuning you want and swap the pipelines to use it: +The default `batch` processor is sized for ClickHouse (`send_batch_size: 10000`, +`timeout: 5s`). If your workload needs lower-latency exports — for example, +latency-sensitive traces or a short-window smoke test that asserts shortly after +emitting — define a new `batch` with the tuning you want and swap the pipelines +to use it: ```yaml # custom.config.yaml @@ -227,23 +274,21 @@ service: Notes: -- You only need to re-declare the pipelines you want to retune; pipelines - you don't mention keep the default `batch` from the base config. In the - example above, `metrics` and `logs/out-rrweb` continue using the default - `batch`. +- You only need to re-declare the pipelines you want to retune; pipelines you + don't mention keep the default `batch` from the base config. In the example + above, `metrics` and `logs/out-rrweb` continue using the default `batch`. - Processor list order matters. Keep `memory_limiter` ahead of the batch processor so back-pressure applies before batching. -- Larger batches are friendlier to ClickHouse (fewer inserts, fewer - MergeTree parts). The defaults — `send_batch_size: 10000`, `timeout: 5s` - — are the recommended starting point; only tune them when you have a - specific reason. +- Larger batches are friendlier to ClickHouse (fewer inserts, fewer MergeTree + parts). The defaults — `send_batch_size: 10000`, `timeout: 5s` — are the + recommended starting point; only tune them when you have a specific reason. - You can combine swap blocks. Define both `memory_limiter/custom` and - `batch/lowlatency`, and reference both in the same pipeline - (e.g. `processors: [memory_limiter/custom, batch/lowlatency]`). + `batch/lowlatency`, and reference both in the same pipeline (e.g. + `processors: [memory_limiter/custom, batch/lowlatency]`). You can verify the new processor is actually running (not just present in -`effective.yaml`) by querying the collector's Prometheus metrics endpoint -on port `8888` and looking for the new `processor` label: +`effective.yaml`) by querying the collector's Prometheus metrics endpoint on +port `8888` and looking for the new `processor` label: ```sh curl -s http://localhost:8888/metrics | grep 'processor="batch/lowlatency"' @@ -253,32 +298,30 @@ curl -s http://localhost:8888/metrics | grep 'processor="batch/lowlatency"' ### Lighter-weight option: env vars for the default `batch` -If you only need to tune `send_batch_size`, `send_batch_max_size`, or -`timeout` on the default `batch` processor and don't need to define a -second processor, the existing env vars also work without a custom config -file: +If you only need to tune `send_batch_size`, `send_batch_max_size`, or `timeout` +on the default `batch` processor and don't need to define a second processor, +the existing env vars also work without a custom config file: - `HYPERDX_OTEL_BATCH_SEND_BATCH_SIZE` (default `10000`) -- `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE` (default `0`, meaning no upper - bound) +- `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE` (default `0`, meaning no upper bound) - `HYPERDX_OTEL_BATCH_TIMEOUT` (default `5s`) -These are read directly from the base `config.yaml`, so they apply -everywhere the default `batch` is referenced. Use the swap pattern when -you need different settings for different pipelines, when you want to -combine batch + memory_limiter changes, or when you want to override -`memory_limiter` (which has no equivalent env-var path). +These are read directly from the base `config.yaml`, so they apply everywhere +the default `batch` is referenced. Use the swap pattern when you need different +settings for different pipelines, when you want to combine batch + +memory_limiter changes, or when you want to override `memory_limiter` (which has +no equivalent env-var path). The same swap pattern works for any other base processor (`transform`, -`resourcedetection`, …) — define a new component with a different name -and re-declare the pipelines that should use it. - -> Pipeline `processors:` lists live in `docker/otel-collector/config.yaml` -> (for OpAMP supervisor mode) and `docker/otel-collector/config.standalone.yaml` -> (for standalone mode). The OpAMP remote config from -> `packages/api/src/opamp/controllers/opampController.ts` intentionally -> does **not** set `processors:` on pipelines, so your bootstrap+custom -> merge is not overwritten. +`resourcedetection`, …) — define a new component with a different name and +re-declare the pipelines that should use it. + +> Pipeline `processors:` lists live in `docker/otel-collector/config.yaml` (for +> OpAMP supervisor mode) and `docker/otel-collector/config.standalone.yaml` (for +> standalone mode). The OpAMP remote config from +> `packages/api/src/opamp/controllers/opampController.ts` intentionally does +> **not** set `processors:` on pipelines, so your bootstrap+custom merge is not +> overwritten. ## Upgrading the OTel Collector version diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index ecd794a456..e84cad931a 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -135,6 +135,9 @@ connectors: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector + v__OTEL_COLLECTOR_VERSION__ extensions: # Core