feat(collector): derive request metrics with trace exemplars from spans

.changeset/spanmetrics-connector.md

@@ -0,0 +1,37 @@
+---
+'@hyperdx/otel-collector': minor
+'@hyperdx/api': minor
+---
+
+feat: derive request metrics with trace exemplars from spans
+
+Adds the OpenTelemetry `spanmetricsconnector` to the collector build and wires
+it into the generated OpAMP config behind `ENABLE_SPAN_METRICS`, off by default.
+The connector reads the traces pipeline and feeds a dedicated metrics pipeline,
+so `traces.span.metrics.*` land in ClickHouse with `Exemplars.*` pointing back
+at the spans they were measured from.
+
+Histogram buckets are exponential rather than a fixed ladder: an explicit ladder
+puts everything slow into one wide top bucket, so a high quantile interpolates
+well past the slowest real request and no exemplar can sit on the plotted line.
+
+Dimensions are limited to `http.route`, `http.request.method` and
+`http.response.status_code` — each bounded by the application's route table or
+by the HTTP spec — and the connector is given an explicit
+`aggregation_cardinality_limit`. Temporality is cumulative, so nothing evicts a
+series once created; one free-form dimension would grow collector memory and the
+ClickHouse write volume without bound.
+
+`ENABLE_SPAN_METRICS_PROM_RW` additionally remote-writes the derived metrics to
+a Prometheus endpoint, for exercising Prometheus's native exemplar path. Because
+the generated config is served from the unauthenticated OpAMP endpoint, an
+endpoint URL carrying `user:token@` credentials is rejected rather than inlined,
+as are non-HTTP schemes. Resource attributes are not promoted to labels on this
+exporter: that happens after the connector's cardinality limit and would send
+host, pod and namespace to a third party.
+
+Requires a collector built from the current `builder-config.yaml`. Roll the
+collector image out before enabling the flag: a config naming a component type
+the binary does not register fails to decode as a whole, and the shipped
+`docker/otel-collector/config.yaml` defines no pipelines of its own, so a
+rejected remote config leaves the collector with nothing to run.

.gitignore

@@ -95,3 +95,7 @@ docker-compose.prod.yml
 
 # webstorm
 .idea
+
+# Stryker mutation-testing output
+packages/*/reports/
+packages/*/.stryker-tmp/

packages/api/src/__tests__/config.test.ts

@@ -59,4 +59,63 @@ describe('config', () => {
       });
     });
   });
+
+  describe('SPAN_METRICS_PROM_RW_ENDPOINT', () => {
+    const ORIGINAL = process.env.SPAN_METRICS_PROM_RW_ENDPOINT;
+
+    afterEach(() => {
+      if (ORIGINAL === undefined) {
+        delete process.env.SPAN_METRICS_PROM_RW_ENDPOINT;
+      } else {
+        process.env.SPAN_METRICS_PROM_RW_ENDPOINT = ORIGINAL;
+      }
+      jest.resetModules();
+    });
+
+    const resolve = (raw: string | undefined) => {
+      if (raw === undefined) {
+        delete process.env.SPAN_METRICS_PROM_RW_ENDPOINT;
+      } else {
+        process.env.SPAN_METRICS_PROM_RW_ENDPOINT = raw;
+      }
+      let resolved: string | undefined;
+      jest.isolateModules(() => {
+        // eslint-disable-next-line @typescript-eslint/no-require-imports, n/no-missing-require
+        resolved = require('@/config').SPAN_METRICS_PROM_RW_ENDPOINT;
+      });
+      return resolved;
+    };
+
+    it('accepts a plain http or https endpoint', () => {
+      expect(resolve('http://prometheus:9090/api/v1/write')).toBe(
+        'http://prometheus:9090/api/v1/write',
+      );
+      expect(resolve('https://prom.example.com/api/v1/push')).toBe(
+        'https://prom.example.com/api/v1/push',
+      );
+    });
+
+    // The generated collector config is served from the unauthenticated OpAMP
+    // endpoint, so a credential in this URL is readable by anyone who can POST
+    // there. Pointing at a hosted Prometheus this way is the common case, which
+    // is why it is rejected rather than passed along.
+    it('rejects an endpoint carrying credentials', () => {
+      expect(resolve('https://user:token@prom.example.com/api/v1/push')).toBe(
+        undefined,
+      );
+      expect(resolve('https://token@prom.example.com/api/v1/push')).toBe(
+        undefined,
+      );
+    });
+
+    it('rejects a non-HTTP scheme or an unparseable URL', () => {
+      expect(resolve('file:///etc/passwd')).toBe(undefined);
+      expect(resolve('prometheus:9090')).toBe(undefined);
+      expect(resolve('not a url')).toBe(undefined);
+    });
+
+    it('is undefined when unset', () => {
+      expect(resolve(undefined)).toBe(undefined);
+    });
+  });
 });

packages/api/src/config.ts

@@ -57,6 +57,46 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES;
 
 export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true';
 
+// Opt-in: have the collector derive request metrics (with trace exemplars) from
+// spans via the spanmetrics connector. Off everywhere by default — set
+// ENABLE_SPAN_METRICS=true to have the telemetry-generator's traces produce
+// coherent metric exemplars end-to-end.
+//
+// Requires a collector built from the current builder-config.yaml. An older
+// image does not register the `spanmetrics` type, and a config naming an
+// unregistered type fails to decode as a whole, so the agent would fall back to
+// no pipelines at all. Roll the collector image out before setting this.
+export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true';
+
+// Opt-in: also remote-write the span-derived metrics (with exemplars) to a
+// Prometheus endpoint so the native Prometheus query_exemplars path can be
+// tested against real data. The endpoint is resolved here (API side) and
+// inlined into the generated collector config, so the collector container does
+// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the
+// endpoint to be set; without it the feature stays disabled.
+//
+// The generated config is served from /v1/opamp, which agents reach without
+// authenticating, so this URL is readable by anyone who can POST there. A URL
+// carrying `user:token@` — the usual way to point at a hosted Prometheus —
+// would hand that credential out with it, so such an endpoint is rejected
+// rather than used. Non-http schemes go the same way: the exporter only speaks
+// HTTP, so anything else is a misconfiguration.
+export const SPAN_METRICS_PROM_RW_ENDPOINT = (() => {
+  const raw = env.SPAN_METRICS_PROM_RW_ENDPOINT;
+  if (!raw) return undefined;
+  let url: URL;
+  try {
+    url = new URL(raw);
+  } catch {
+    return undefined;
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') return undefined;
+  if (url.username || url.password) return undefined;
+  return raw;
+})();
+export const IS_SPAN_METRICS_PROM_RW_ENABLED =
+  env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT;
+
 // FOR CI ONLY
 export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
 export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;