diff --git a/.changeset/query-exemplars-route.md b/.changeset/query-exemplars-route.md new file mode 100644 index 0000000000..e4a43c2d83 --- /dev/null +++ b/.changeset/query-exemplars-route.md @@ -0,0 +1,27 @@ +--- +'@hyperdx/api': minor +--- + +feat: add /v1/prometheus/query_exemplars, and harden the Prometheus proxy + +Adds a `query_exemplars` route that proxies to Prometheus's native +`/api/v1/query_exemplars` for Prometheus-backed connections, and answers with an +empty success for ClickHouse-backed ones, where exemplars are read from the metric +table instead. + +Three fixes to the shared proxy while adding a route to it: + +- Responses now carry `X-Content-Type-Options: nosniff`, set before anything can + return so the proxy's own error bodies get it too, and the upstream content-type + is never forwarded — every response is relabelled `application/json`. The + connection host is member-configured, so its response body is untrusted output on + our own origin, and an allowlist is easy to slip past: `application/json, + text/html` clears a prefix-anchored JSON test while the browser keeps the last + media type. +- A client that navigates away mid-body no longer counts as a backend error. +- Proxy failures increment `prometheusQueryErrors`. `proxyToPrometheus` handles its + own failures and returns normally, so the callers' `catch` never ran and all four + proxied endpoints reported zero errors while still recording duration. Counted on + 5xx only, so a user's malformed PromQL does not read as a backend fault. +- The exemplar window is bounded by narrowing rather than rejecting, so a wide + dashboard range still works. diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts index dc48dd8499..e5703f8ca6 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts @@ -3,13 +3,20 @@ import { Types } from 'mongoose'; import * as config from '@/config'; import { getAgent, getLoggedInAgent, getServer } from '@/fixtures'; import Connection from '@/models/connection'; +import { PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC } from '@/routers/api/prometheus'; const mockFetch = global.fetch as jest.Mock; // The proxy now streams the upstream response straight through (no // `await resp.json()`), so test mocks must expose the fields the pipeline // actually reads: `status`, `headers.get()`, and a web `ReadableStream` body. -function fakeUpstreamResponse(payload: unknown, status = 200) { +// Returned as `Response` so callers can hand it straight to `mockResolvedValue` +// without an `as any` at every site — the proxy only touches the fields below. +function fakeUpstreamResponse( + payload: unknown, + status = 200, + contentType = 'application/json', +): Response { const body = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(JSON.stringify(payload))); @@ -19,11 +26,11 @@ function fakeUpstreamResponse(payload: unknown, status = 200) { return { ok: status >= 200 && status < 300, status, - headers: new Headers({ 'content-type': 'application/json' }), + headers: new Headers({ 'content-type': contentType }), body, text: jest.fn().mockResolvedValue(JSON.stringify(payload)), json: jest.fn().mockResolvedValue(payload), - }; + } as unknown as Response; } describe('prometheus router', () => { @@ -36,18 +43,21 @@ describe('prometheus router', () => { afterEach(async () => { await server.clearDBs(); mockFetch.mockReset(); - mockFetch.mockResolvedValue(fakeUpstreamResponse({}) as any); + mockFetch.mockResolvedValue(fakeUpstreamResponse({})); }); afterAll(async () => { await server.stop(); }); - const seedPrometheusConnection = async (teamId: Types.ObjectId) => { + const seedPrometheusConnection = async ( + teamId: Types.ObjectId, + host = 'http://prom.example.com', + ) => { return Connection.create({ team: teamId, name: 'Prom', - host: 'http://prom.example.com', + host, username: '', password: '', isPrometheusEndpoint: true, @@ -111,6 +121,151 @@ describe('prometheus router', () => { }); }); + // The connection host is member-configured, so its response body is untrusted + // output on our own origin: /api/* is same-origin-proxied by the app and the + // session cookie is sameSite lax. A text/html body forwarded verbatim would + // render as script here. + it('never forwards a non-JSON upstream content-type, and always sends nosniff', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse( + '', + 200, + 'text/html', + ), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.headers['content-type']).toContain('application/json'); + expect(res.headers['content-type']).not.toContain('text/html'); + expect(res.headers['x-content-type-options']).toBe('nosniff'); + }); + + // The reason the content-type is relabelled rather than allowlisted. A + // prefix-anchored JSON test passes this value — the comma is a word boundary + // — but the browser's MIME extraction keeps the *last* essence, so the body + // would render as HTML on our own origin. `Headers.get()` also joins two + // separate Content-Type headers into exactly this shape. + it('does not forward a JSON content-type that carries a second media type', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse( + '', + 200, + 'application/json, text/html', + ), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.headers['content-type']).not.toContain('text/html'); + expect(res.headers['content-type']).toBe( + 'application/json; charset=utf-8', + ); + expect(res.headers['x-content-type-options']).toBe('nosniff'); + }); + + // A distinguishable JSON flavour, so this cannot pass by coinciding with the + // fallback: the upstream value must not survive even when it is valid JSON. + it('relabels even a legitimate non-standard JSON content-type', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse( + { status: 'success', data: [] }, + 200, + 'application/vnd.api+json', + ), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.headers['content-type']).toBe( + 'application/json; charset=utf-8', + ); + expect(res.headers['content-type']).not.toContain('vnd.api'); + }); + + // The unreachable-upstream path builds its message from the target URL, so + // it needs nosniff too — and must not echo basic-auth credentials from a + // connection host into a body the browser shows. + it('sends nosniff on a 502 and redacts credentials from the message', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection( + team._id, + 'http://user:s3cr3t@prom.example.com', + ); + + mockFetch.mockRejectedValueOnce( + Object.assign(new Error('fetch failed'), { + cause: { code: 'ECONNREFUSED' }, + }), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(502); + + expect(res.headers['x-content-type-options']).toBe('nosniff'); + expect(res.body.error).toContain('ECONNREFUSED'); + expect(res.body.error).not.toContain('s3cr3t'); + }); + + // nosniff is set by router middleware, so the helper's own error bodies — + // which echo the caller-supplied host — carry it too. + it('sends nosniff on its own error responses', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id, 'not-a-valid-url'); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(400); + + expect(res.headers['x-content-type-options']).toBe('nosniff'); + }); + it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); const conn = await seedPrometheusConnection(team._id); @@ -119,9 +274,7 @@ describe('prometheus router', () => { status: 'success', data: { resultType: 'matrix', result: [] }, }; - mockFetch.mockResolvedValueOnce( - fakeUpstreamResponse(promResponse) as any, - ); + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/query_range') @@ -220,9 +373,7 @@ describe('prometheus router', () => { status: 'success', data: { resultType: 'vector', result: [] }, }; - mockFetch.mockResolvedValueOnce( - fakeUpstreamResponse(promResponse) as any, - ); + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/query') @@ -256,9 +407,7 @@ describe('prometheus router', () => { const conn = await seedPrometheusConnection(team._id); const promResponse = { status: 'success', data: ['up', 'requests'] }; - mockFetch.mockResolvedValueOnce( - fakeUpstreamResponse(promResponse) as any, - ); + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/label/__name__/values') @@ -270,4 +419,156 @@ describe('prometheus router', () => { expect(calledUrl).toContain('/api/v1/label/__name__/values'); }); }); + + describe('GET /v1/prometheus/query_exemplars', () => { + it('returns 400 when query parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('query'), + }); + }); + + it('returns 400 when connectionId is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ query: 'up' }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('connectionId'), + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('returns 404 for a connection owned by another team', async () => { + const { agent } = await getLoggedInAgent(server); + // A real, resolvable connection — just not this team's. The 404 must come + // from the team scoping, not from the id simply not existing. + const otherTeamConn = await seedPrometheusConnection( + new Types.ObjectId(), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ query: 'up', connectionId: otherTeamConn._id.toString() }) + .expect(404); + expect(res.body).toMatchObject({ + status: 'error', + error: 'Connection not found', + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { status: 'success', data: [] }; + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual(promResponse); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('/api/v1/query_exemplars'); + expect(calledUrl).toContain('query=up'); + }); + + // The existing case above uses a 60-second window, where the Math.max in + // resolveExemplarWindow is a no-op — so it would still pass if the narrowed + // window never reached the outgoing URL. This drives a range wide enough for + // the clamp to bite and inspects what was actually requested. + it('narrows an over-wide window on the outgoing request and keeps end', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse({ status: 'success', data: [] }), + ); + + const end = 1700000000; + const thirtyDays = 30 * 24 * 60 * 60; + + await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: String(end - thirtyDays), + end: String(end), + connectionId: conn._id.toString(), + }) + .expect(200); + + const requested = new URL(mockFetch.mock.calls[0][0] as string); + const sentStart = Number(requested.searchParams.get('start')); + expect(Number(requested.searchParams.get('end'))).toBe(end); + expect(sentStart).toBeGreaterThan(end - thirtyDays); + expect(end - sentStart).toBe(PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC); + }); + + it('returns an empty result for ClickHouse-backed connections (no native exemplar table function)', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedClickHouseConnection(team._id); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual({ status: 'success', data: [] }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + // The two backends have different required-parameter contracts on one route, + // keyed on an attribute the caller does not control. That is deliberate — the + // ClickHouse branch does no upstream work, so 400ing it would light up the + // chart's exemplar error indicator on a healthy chart — but it is surprising + // enough to pin rather than leave to a comment. + it('validates start/end only on the branch that reaches Prometheus', async () => { + const { agent, team } = await getLoggedInAgent(server); + const prom = await seedPrometheusConnection(team._id); + const ch = await seedClickHouseConnection(team._id); + + await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: 'not-a-time', + connectionId: prom._id.toString(), + }) + .expect(400); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: 'not-a-time', + connectionId: ch._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual({ status: 'success', data: [] }); + }); + }); }); diff --git a/packages/api/src/routers/api/__tests__/prometheus.test.ts b/packages/api/src/routers/api/__tests__/prometheus.test.ts index 6f5b0b7063..4fa1848083 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.test.ts @@ -1,8 +1,21 @@ +const mockCounterAdd = jest.fn(); + +jest.mock('@/utils/instrumentation', () => { + const actual = jest.requireActual('@/utils/instrumentation'); + return { + ...actual, + getCounter: () => ({ add: mockCounterAdd }), + }; +}); + import { formatMatrixResponse, formatVectorResponse, + isClientDisconnect, parseDuration, parseTimestamp, + recordProxyOutcome, + resolveExemplarWindow, } from '@/routers/api/prometheus'; describe('parseTimestamp', () => { @@ -133,3 +146,114 @@ describe('formatVectorResponse', () => { ]); }); }); + +describe('resolveExemplarWindow', () => { + const DAY = 24 * 60 * 60; + const end = 1_700_000_000; + + it('passes a window inside the cap through untouched', () => { + expect(resolveExemplarWindow(String(end - DAY), String(end))).toEqual({ + start: end - DAY, + end, + }); + }); + + it('narrows an over-wide window instead of rejecting it', () => { + // A 30-day dashboard range is an ordinary request, and Prometheus keeps + // exemplars in a small recent buffer, so the older part has nothing to + // return. Rejecting would surface as the chart's exemplar error indicator on + // a perfectly healthy chart. + const result = resolveExemplarWindow(String(end - 30 * DAY), String(end)); + expect(result).toEqual({ start: end - 7 * DAY, end }); + }); + + it('keeps the requested end when narrowing', () => { + const result = resolveExemplarWindow(String(end - 30 * DAY), String(end)); + expect('end' in result && result.end).toBe(end); + }); + + it('rejects an inverted range', () => { + expect(resolveExemplarWindow(String(end), String(end - DAY))).toEqual({ + error: 'invalid or missing start/end parameters', + }); + }); + + it('rejects a missing or unparseable bound', () => { + for (const [start, endArg] of [ + [undefined, String(end)], + [String(end - DAY), undefined], + ['', String(end)], + ['not-a-time', String(end)], + ] as [string | undefined, string | undefined][]) { + expect(resolveExemplarWindow(start, endArg)).toEqual({ + error: 'invalid or missing start/end parameters', + }); + } + }); + + it('accepts an ISO timestamp, matching parseTimestamp', () => { + const result = resolveExemplarWindow( + '2023-11-14T22:13:20Z', + '2023-11-14T22:14:20Z', + ); + expect('start' in result).toBe(true); + }); + + it('honours an explicit cap', () => { + expect(resolveExemplarWindow(String(end - 100), String(end), 10)).toEqual({ + start: end - 10, + end, + }); + }); +}); + +describe('recordProxyOutcome', () => { + beforeEach(() => { + mockCounterAdd.mockClear(); + }); + + // proxyToPrometheus handles its own failures by writing a status and returning + // normally, so the caller's catch never sees an upstream outage — this is the + // only place the proxy path's error counter moves. + it.each([500, 502, 503, 504])('counts an upstream %i', status => { + recordProxyOutcome(status, 'query_exemplars', 'prometheus'); + expect(mockCounterAdd).toHaveBeenCalledWith(1, { + endpoint: 'query_exemplars', + backend: 'prometheus', + }); + }); + + // A 4xx is almost always a PromQL expression the user typed. Counting those + // would make the counter track typos rather than backend health, which is what + // alerts and SLOs read it for. + it.each([200, 204, 400, 404, 422])('does not count a %i', status => { + recordProxyOutcome(status, 'query_range', 'prometheus'); + expect(mockCounterAdd).not.toHaveBeenCalled(); + }); +}); + +describe('isClientDisconnect', () => { + // `pipeline` destroys the response stream before rejecting no matter which end + // failed, so `res.destroyed` cannot tell these apart — an earlier revision + // tested it and silently reclassified every upstream fault as a cancellation, + // leaving the error counter at zero for truncated bodies. + it('is false when the upstream stream fails mid-body', () => { + // What undici raises when the upstream resets: no `code`. + expect(isClientDisconnect(new TypeError('terminated'))).toBe(false); + }); + + it('is true when the client hangs up mid-body', () => { + expect( + isClientDisconnect( + Object.assign(new Error('Premature close'), { + code: 'ERR_STREAM_PREMATURE_CLOSE', + }), + ), + ).toBe(true); + }); + + it('is false for a non-Error rejection', () => { + expect(isClientDisconnect('boom')).toBe(false); + expect(isClientDisconnect(undefined)).toBe(false); + }); +}); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 07f446816f..a0edd9c45f 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -33,6 +33,14 @@ const prometheusQueryErrors = getCounter('hyperdx.prometheus.query_errors', { // Accept URL-encoded form bodies (Prometheus standard) and JSON router.use(express.urlencoded({ extended: true })); +// Every response from this router either streams a member-configured upstream or +// echoes caller-supplied text back in an error body. Set here rather than in the +// proxy helper so the handlers' own catch blocks are covered too. +router.use((_req, res, next) => { + res.setHeader('x-content-type-options', 'nosniff'); + next(); +}); + // -------------------------- // Param parsing helpers // -------------------------- @@ -147,20 +155,45 @@ const PROMETHEUS_CH_TIMEOUT_MS = 30_000; const PROMETHEUS_MAX_EXECUTION_SEC = 30; const PROMETHEUS_MAX_RESULT_ROWS = '100000'; const PROMETHEUS_MAX_RESOLUTION = 11_000; +// Widest window /query_exemplars will proxy. Prometheus's exemplar store is a +// small circular buffer, so a wider range mostly costs a bigger streamed body +// for no extra markers — the chart's own thinning caps what's rendered anyway. +export const PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC = 7 * 24 * 60 * 60; + +/** + * Whether a rejection from streaming the upstream body means the client hung up + * rather than the backend failing. + * + * The error code is the only usable signal. `pipeline` destroys every stream it + * touches before rejecting — the destination included, whichever end actually + * failed — so `res.destroyed` is true either way and testing it would classify + * every upstream fault as a user cancellation. Node also resolves the race in + * our favour: a real error always beats the ERR_STREAM_PREMATURE_CLOSE that the + * cascading destroy raises, never the other way round. + */ +export function isClientDisconnect(err: unknown): boolean { + return ( + err instanceof Error && + (err as NodeJS.ErrnoException).code === 'ERR_STREAM_PREMATURE_CLOSE' + ); +} // Forwards the response straight from the upstream Prometheus to the -// HyperDX client. The response can be multi-megabyte (e.g. `/label/__name__/ +// HyperDX client. Returns the HTTP status it wrote, so callers can record an +// error metric: this helper handles its own failures by writing 400/502/504 and +// returning normally, so a caller's `catch` never sees an upstream outage and +// would otherwise report zero errors while still recording duration. The response can be multi-megabyte (e.g. `/label/__name__/ // values` on a large Prometheus), so we avoid `await resp.json()` + // `res.json(...)` which would parse + re-serialize the whole body in memory. // Prometheus's native response shape (`{status, data}` / `{status, errorType, -// error}`) is already what HyperDX clients expect, so we forward the status -// code and content-type as-is. +// error}`) is already what HyperDX clients expect, so we forward the status code +// as-is — but never the content-type, which is always relabelled (see below). async function proxyToPrometheus( upstreamHost: string, path: string, params: Record, res: express.Response, -): Promise { +): Promise { let url: URL; try { url = new URL(path, upstreamHost); @@ -170,7 +203,7 @@ async function proxyToPrometheus( errorType: 'bad_data', error: `Connection host is not a valid URL: ${JSON.stringify(upstreamHost)}`, }); - return; + return 400; } for (const [k, v] of Object.entries(params)) { if (['connectionId', 'database', 'table'].includes(k)) continue; @@ -178,6 +211,16 @@ async function proxyToPrometheus( } const target = url.toString(); + // A connection host may carry basic-auth credentials (`http://user:pw@host`), + // and the error bodies below are shown in the browser. Strip them for display + // only — `target` itself still needs them to authenticate. + const redactedTarget = (() => { + const safe = new URL(url); + safe.username = ''; + safe.password = ''; + return safe.toString(); + })(); + let upstreamResp: Response; try { upstreamResp = await fetch(target, { @@ -188,9 +231,9 @@ async function proxyToPrometheus( res.status(504).json({ status: 'error', errorType: 'timeout', - error: `Prometheus request to ${target} timed out after ${PROMETHEUS_PROXY_TIMEOUT_MS}ms`, + error: `Prometheus request to ${redactedTarget} timed out after ${PROMETHEUS_PROXY_TIMEOUT_MS}ms`, }); - return; + return 504; } // Node's fetch wraps the real network error (ECONNREFUSED, ENOTFOUND, TLS, // …) in `.cause`. Without unwrapping it the client only sees a useless @@ -208,29 +251,70 @@ async function proxyToPrometheus( res.status(502).json({ status: 'error', errorType: 'unavailable', - error: `Failed to reach Prometheus at ${target} (${detail})`, + error: `Failed to reach Prometheus at ${redactedTarget} (${detail})`, }); - return; + return 502; } res.status(upstreamResp.status); - const contentType = upstreamResp.headers.get('content-type'); - if (contentType) res.setHeader('content-type', contentType); + + // The connection host is member-configured, so its response is untrusted output + // on our own origin: the app same-origin-proxies /api/*, and the session cookie + // is sameSite lax. Forwarding the upstream content-type would let a text/html + // body render as script here, so it is never forwarded — every response is + // relabelled, which keeps a genuine Prometheus error body readable while making + // a hostile one inert. + // + // An allowlist was tried first and is not worth it. Prometheus only ever + // answers application/json, so passing anything through buys nothing, and + // getting the check right is easy to botch: `Content-Type: application/json, + // text/html` (which is also what Headers.get() produces from two separate + // headers) passes a prefix-anchored JSON test, while the browser's MIME + // extraction keeps the *last* essence — text/html. + res.setHeader('content-type', 'application/json; charset=utf-8'); if (!upstreamResp.body) { res.end(); - return; + return upstreamResp.status; } try { await pipeline(Readable.fromWeb(upstreamResp.body as any), res); } catch (err) { + // A client that navigates away mid-body makes `pipeline` reject too. That is + // not a backend failure, and returning 502 for it would have the caller count + // ordinary user cancellations against Prometheus's health. Report the + // upstream's own status in that case; only a genuine stream failure is 502. + const clientGone = isClientDisconnect(err); + // Headers are already sent at this point — best we can do is destroy the // socket so the client sees a truncated response instead of a hung // connection. if (!res.writableEnded) { res.destroy(err instanceof Error ? err : new Error(String(err))); } + return clientGone ? upstreamResp.status : 502; + } + return upstreamResp.status; +} + +/** + * Records an error for a proxied response that failed server-side. + * proxyToPrometheus never throws, so this is the only place the error counter + * gets incremented on the proxy path. + * + * 5xx only. An upstream 4xx is almost always a malformed PromQL expression the + * user typed, and counting those would make this counter track user typos rather + * than backend health — which is what alerts and SLOs read it for. The helper's + * own failure statuses (502 unreachable, 504 timeout) are 5xx and so are counted. + */ +export function recordProxyOutcome( + status: number, + endpoint: string, + backend: PrometheusBackend, +) { + if (status >= 500) { + prometheusQueryErrors.add(1, { endpoint, backend }); } } @@ -281,12 +365,13 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { // directly to connection.host instead of running a ClickHouse query. if (connection.isPrometheusEndpoint) { backend = 'prometheus'; - await proxyToPrometheus( + const status = await proxyToPrometheus( connection.host, '/api/v1/query_range', params, res, ); + recordProxyOutcome(status, 'query_range', backend); return; } @@ -414,7 +499,13 @@ const queryHandler: express.RequestHandler = async (req, res) => { if (connection.isPrometheusEndpoint) { backend = 'prometheus'; - await proxyToPrometheus(connection.host, '/api/v1/query', params, res); + const status = await proxyToPrometheus( + connection.host, + '/api/v1/query', + params, + res, + ); + recordProxyOutcome(status, 'query', backend); return; } @@ -478,6 +569,146 @@ const queryHandler: express.RequestHandler = async (req, res) => { router.get('/query', queryHandler); router.post('/query', queryHandler); +/** + * Resolve the exemplar window a /query_exemplars request should be proxied with. + * + * Returns either the bounded window or the reason it is unusable, so the handler + * stays a thin translation to HTTP and this logic can be tested without a route. + * Over-wide windows are narrowed rather than rejected: a 30d dashboard range is an + * ordinary request, and Prometheus keeps exemplars in a small recent buffer, so + * the older part has nothing to return anyway. + */ +export function resolveExemplarWindow( + rawStart: string | undefined, + rawEnd: string | undefined, + maxWindowSec = PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC, +): { start: number; end: number } | { error: string } { + const parse = (v: string | undefined) => { + if (v == null || v === '') return NaN; + try { + return parseTimestamp(v); + } catch { + return NaN; + } + }; + const start = parse(rawStart); + const end = parse(rawEnd); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { + return { error: 'invalid or missing start/end parameters' }; + } + return { start: Math.max(start, end - maxWindowSec), end }; +} + +// -------------------------- +// GET|POST /query_exemplars +// -------------------------- + +// Native Prometheus exposes exemplars via /api/v1/query_exemplars. We proxy +// straight through for Prometheus-backed connections. ClickHouse-backed metric +// exemplars come from the OTel metric tables' own `Exemplars.*` columns, read by +// the chart query the app already issues — there is no ClickHouse table function +// to call here, so this returns an empty result rather than a second query. +const queryExemplarsHandler: express.RequestHandler = async (req, res) => { + const startedAt = performance.now(); + let backend: PrometheusBackend = 'unknown'; + try { + const { teamId } = getNonNullUserWithTeam(req); + const params = getParams(req); + + const query = params.query; + if (!query) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: query', + }); + } + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + // No `selectPassword` — neither branch builds a ClickhouseClient, so there + // is no reason to pull the connection secret into request scope. + const connection = await getConnectionById(teamId.toString(), connectionId); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + if (connection.isPrometheusEndpoint) { + backend = 'prometheus'; + + // Bound the requested window before proxying. Unlike /query_range there is + // no `step` to cap the result size with, and Prometheus's /query_exemplars + // takes no limit parameter — the window is the only lever, and the response + // streams through this process. Parsed (not just forwarded) so a malformed + // timestamp fails here rather than upstream. + // + // Deliberately inside this branch: the ClickHouse-backed branch below never + // reaches Prometheus and answers with an empty success, so bounding it there + // would turn a healthy wide-range chart into a 400 (and light up the chart's + // exemplar error indicator) for a request that does no upstream work. + // + // parseTimestamp throws on a missing/unparseable value; a bad client param + // is a 400, not an upstream error, so it is resolved here, not in the catch. + const window = resolveExemplarWindow(params.start, params.end); + if ('error' in window) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: window.error, + }); + } + + // Both bounds come from the resolved window, not just `start`. Forwarding + // the raw `end` would let a value this function accepts but Prometheus + // rejects — leading whitespace, a `0x` literal — be declared valid here + // and then 400 upstream. + const status = await proxyToPrometheus( + connection.host, + '/api/v1/query_exemplars', + { + ...params, + start: String(window.start), + end: String(window.end), + }, + res, + ); + recordProxyOutcome(status, 'query_exemplars', backend); + return; + } + + // ClickHouse-backed PromQL: no native exemplar table function. Exemplars + // for structured metric charts are fetched app-side from the metric table. + backend = 'clickhouse'; + return res.json({ status: 'success', data: [] }); + } catch (e) { + prometheusQueryErrors.add(1, { endpoint: 'query_exemplars', backend }); + logger.error(e, 'Prometheus query_exemplars error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } finally { + prometheusQueryDuration.record(performance.now() - startedAt, { + endpoint: 'query_exemplars', + backend, + }); + } +}; +router.get('/query_exemplars', queryExemplarsHandler); +router.post('/query_exemplars', queryExemplarsHandler); + // -------------------------- // GET /label/:name/values // -------------------------- @@ -528,12 +759,13 @@ router.get('/label/:name/values', async (req, res) => { // Proxy to Prometheus if endpoint is set if (connection.isPrometheusEndpoint) { backend = 'prometheus'; - await proxyToPrometheus( + const status = await proxyToPrometheus( connection.host, `/api/v1/label/${labelName}/values`, params, res, ); + recordProxyOutcome(status, 'label_values', backend); return; }