feat(api): serve PromQL label lookups from ClickHouse's Prometheus HTTP API

.changeset/promql-labels-via-clickhouse-http-api.md

@@ -0,0 +1,8 @@
+---
+'@hyperdx/api': minor
+'@hyperdx/common-utils': minor
+---
+
+feat: serve PromQL label lookups from ClickHouse's Prometheus HTTP API
+
+On ClickHouse 26.9+ with the `prometheus_api_v1` handler, `/labels` and `/label/:name/values` for a ClickHouse-backed PromQL source now proxy to the handler instead of querying the TimeSeries tags table with SQL. Older servers, servers without the handler, and sources set to `table-function` keep the SQL path, so a source that runs queries over HTTP on 26.6-26.8 still gets label autocomplete.

agent_docs/observability.md

@@ -305,9 +305,10 @@ calls `/api/v1/prometheus/query{,_range}`, and `/labels` and
 (`QUERY_ATTRIBUTION_HEADER`), and the router parses it with `parseLogComment`
 into a `withQueryAttribution` scope layered over the auth middleware's `api`
 scope. The browser's surface and ids win; `trace` and `label` stay the
-server's. Queries the route runs through a ClickHouse client (label lookups,
-and the `prometheusQuery()`/`prometheusQueryRange()` fallback) pick the scope
-up as usual. Requests proxied to ClickHouse's Prometheus HTTP API on 26.6+ are tagged
+server's. Queries the route runs through a ClickHouse client (label lookups
+before 26.9, and the `prometheusQuery()`/`prometheusQueryRange()` fallback)
+pick the scope up as usual. Requests proxied to ClickHouse's Prometheus HTTP
+API (queries on 26.6+, label lookups on 26.9+) are tagged
 by `clickhousePrometheusRequest`: `log_comment` as a URL setting, `query_id` as
 the `X-ClickHouse-Query-Id` header, because the handler rejects `query_id` as an
 unknown setting in the URL.

packages/api/src/controllers/__tests__/promqlBackend.test.ts

@@ -2,17 +2,24 @@ import type { PromqlSourceBackend } from '@hyperdx/common-utils/dist/types';
 
 import {
   clearBackendProbeCache,
+  resolveClickhouseLabelBackend,
   resolveClickhouseQueryBackend,
 } from '@/controllers/promql';
-import { connectionSupportsPrometheusHttpApi } from '@/controllers/timeseriesEngine';
+import { connectionPrometheusHttpApiSupport } from '@/controllers/timeseriesEngine';
 
 jest.mock('@/controllers/timeseriesEngine', () => ({
   ...jest.requireActual('@/controllers/timeseriesEngine'),
-  connectionSupportsPrometheusHttpApi: jest.fn(),
+  connectionPrometheusHttpApiSupport: jest.fn(),
 }));
 
 const mockFetch = jest.mocked(global.fetch);
-const mockVersionCheck = jest.mocked(connectionSupportsPrometheusHttpApi);
+const mockVersionCheck = jest.mocked(connectionPrometheusHttpApiSupport);
+
+// What the version check answers for a server on each side of the 26.6 (query)
+// and 26.9 (label lookup) cut-offs.
+const CLICKHOUSE_26_10 = { query: true, labels: true };
+const CLICKHOUSE_26_8 = { query: true, labels: false };
+const CLICKHOUSE_26_5 = { query: false, labels: false };
 
 const handlerAnswer = () =>
   new Response(JSON.stringify({ status: 'success', data: 'up' }));
@@ -46,7 +53,7 @@ describe('resolveClickhouseQueryBackend', () => {
   beforeEach(() => {
     clearBackendProbeCache();
     mockVersionCheck.mockReset();
-    mockVersionCheck.mockResolvedValue(true);
+    mockVersionCheck.mockResolvedValue(CLICKHOUSE_26_10);
     mockFetch.mockReset();
   });
 
@@ -74,7 +81,7 @@ describe('resolveClickhouseQueryBackend', () => {
     });
 
     it('keeps a server that is too old without probing the handler', async () => {
-      mockVersionCheck.mockResolvedValue(false);
+      mockVersionCheck.mockResolvedValue(CLICKHOUSE_26_5);
 
       await expect(resolveKind()).resolves.toBe('table-function');
       await expect(resolveKind()).resolves.toBe('table-function');
@@ -196,3 +203,65 @@ describe('resolveClickhouseQueryBackend', () => {
     });
   });
 });
+
+describe('resolveClickhouseLabelBackend', () => {
+  const resolveLabels = (promqlBackend?: PromqlSourceBackend) =>
+    resolveClickhouseLabelBackend(
+      // @ts-expect-error mock connection carries only the fields the resolver reads
+      target(undefined, promqlBackend),
+    );
+
+  beforeEach(() => {
+    clearBackendProbeCache();
+    mockVersionCheck.mockReset();
+    mockFetch.mockReset();
+    mockFetch.mockImplementation(async () => handlerAnswer());
+  });
+
+  it.each([[undefined], ['auto' as const], ['clickhouse-http' as const]])(
+    'uses the HTTP API on 26.9+ with the handler when the setting is %s',
+    async setting => {
+      mockVersionCheck.mockResolvedValue(CLICKHOUSE_26_10);
+
+      await expect(resolveLabels(setting)).resolves.toBe('clickhouse-http');
+    },
+  );
+
+  // 26.6-26.8 answers queries over HTTP but not label lookups, so a source on
+  // the HTTP API must not lose its labels there.
+  it.each([[undefined], ['clickhouse-http' as const]])(
+    'reads labels with SQL on 26.8 even though queries use the HTTP API (setting %s)',
+    async setting => {
+      mockVersionCheck.mockResolvedValue(CLICKHOUSE_26_8);
+
+      await expect(resolveLabels(setting)).resolves.toBe('sql');
+      await expect(resolveKind(undefined, setting)).resolves.toBe(
+        'clickhouse-http',
+      );
+    },
+  );
+
+  it('reads labels with SQL when the server has no handler', async () => {
+    mockVersionCheck.mockResolvedValue(CLICKHOUSE_26_10);
+    mockFetch.mockImplementation(async () => noHandlerAnswer());
+
+    await expect(resolveLabels()).resolves.toBe('sql');
+  });
+
+  it('reads labels with SQL for a table-function source, without a probe', async () => {
+    await expect(resolveLabels('table-function')).resolves.toBe('sql');
+
+    expect(mockVersionCheck).not.toHaveBeenCalled();
+    expect(mockFetch).not.toHaveBeenCalled();
+  });
+
+  it('shares the kept probe answer with queries', async () => {
+    mockVersionCheck.mockResolvedValue(CLICKHOUSE_26_10);
+
+    await resolveKind();
+    await expect(resolveLabels()).resolves.toBe('clickhouse-http');
+
+    expect(mockVersionCheck).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+  });
+});

packages/api/src/controllers/__tests__/timeseriesEngine.test.ts

@@ -1,8 +1,41 @@
+import { ClickHouseError } from '@clickhouse/client-common';
+import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
+
 import {
   formatMatrixResponse,
   formatVectorResponse,
+  isNoTimeRangesTableError,
 } from '@/controllers/timeseriesEngine';
 
+describe('isNoTimeRangesTableError', () => {
+  // The shape ClickhouseClient throws: the server's error wrapped as `cause`.
+  const queryError = (type: string) =>
+    Object.assign(new ClickHouseQueryError('query failed', 'DESCRIBE ...'), {
+      cause: new ClickHouseError({ message: 'failed', code: '0', type }),
+    });
+
+  it.each([
+    ['UNKNOWN_FUNCTION', 'servers before 26.10'],
+    ['UNKNOWN_TABLE', '26.10 and later'],
+  ])('accepts %s (%s)', type => {
+    expect(isNoTimeRangesTableError(queryError(type))).toBe(true);
+  });
+
+  it('rejects any other ClickHouse error', () => {
+    expect(isNoTimeRangesTableError(queryError('TIMEOUT_EXCEEDED'))).toBe(
+      false,
+    );
+  });
+
+  it('rejects an error without a ClickHouse type, whatever its message', () => {
+    expect(
+      isNoTimeRangesTableError(
+        new Error('Unknown table function timeSeriesTimeRanges'),
+      ),
+    ).toBe(false);
+  });
+});
+
 describe('formatMatrixResponse', () => {
   it('converts prometheusQueryRange rows into Prometheus matrix shape', () => {
     const rows = [

packages/api/src/controllers/promql.ts

@@ -18,9 +18,10 @@ import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import {
-  connectionSupportsPrometheusHttpApi,
+  connectionPrometheusHttpApiSupport,
   PROMETHEUS_MAX_EXECUTION_SEC,
   PROMETHEUS_MAX_RESULT_ROWS,
+  type PrometheusHttpApiSupport,
   queryInstantViaTableFunction,
   queryLabelNames,
   queryLabelValues,
@@ -486,11 +487,17 @@ export function clearBackendProbeCache(): void {
 /** The probe could not tell, so its outcome must not be kept. */
 class InconclusiveProbeError extends Error {}
 
+const NO_PROMETHEUS_HTTP_API: PrometheusHttpApiSupport = {
+  query: false,
+  labels: false,
+};
+
 /**
- * Asks a ClickHouse connection whether it answers PromQL over HTTP. Two
- * conditions, cheapest first: the server is new enough (26.6+), and the
- * prometheus_api_v1 handler is actually configured — a self-managed server
- * needs an `<http_handlers>` rule for that, which a stock install lacks. The
+ * Asks a ClickHouse connection which parts of the Prometheus HTTP API it
+ * serves. Two conditions, cheapest first: the server is new enough (26.6+ for
+ * queries, 26.9+ for label lookups), and the prometheus_api_v1 handler is
+ * actually configured — a self-managed server needs an `<http_handlers>` rule
+ * for that, which a stock install lacks. The
  * handler answers every request, error or not, with Prometheus JSON
  * (`{status, ...}`); a server without it answers a plain-text 404.
  * `format_query` is used as the probe because it is the one endpoint that
@@ -504,13 +511,13 @@ class InconclusiveProbeError extends Error {}
 async function probePrometheusHttpApi(
   client: ClickhouseClient,
   connection: { id: string; host: string; username: string; password?: string },
-): Promise<boolean> {
-  const supported = await connectionSupportsPrometheusHttpApi({
+): Promise<PrometheusHttpApiSupport> {
+  const supported = await connectionPrometheusHttpApiSupport({
     client,
     connectionId: connection.id,
   });
   if (supported === undefined) throw new InconclusiveProbeError();
-  if (!supported) return false;
+  if (!supported.query) return NO_PROMETHEUS_HTTP_API;
 
   let status: number;
   let text: string;
@@ -531,27 +538,27 @@ async function probePrometheusHttpApi(
     throw new InconclusiveProbeError();
   }
 
-  if (typeof parseJSON(text)?.status === 'string') return true;
-  if (status === 404) return false;
+  if (typeof parseJSON(text)?.status === 'string') return supported;
+  if (status === 404) return NO_PROMETHEUS_HTTP_API;
   throw new InconclusiveProbeError();
 }
 
 /**
- * Whether a ClickHouse connection answers PromQL over HTTP, keeping a
- * definitive answer for BACKEND_PROBE_TTL_MS so each query does not pay for a
- * `SELECT version()` and a probe request first. Keyed on the host as well as
- * the connection, so editing a connection's host drops its answer. Concurrent
- * callers share one probe. An inconclusive probe is answered "no" for this
- * request only.
+ * Which parts of the Prometheus HTTP API a ClickHouse connection serves,
+ * keeping a definitive answer for BACKEND_PROBE_TTL_MS so each request does
+ * not pay for a `SELECT version()` and a probe request first. Keyed on the
+ * host as well as the connection, so editing a connection's host drops its
+ * answer. Concurrent callers share one probe. An inconclusive probe is
+ * answered "none" for this request only.
  */
-async function clickhouseServesPrometheusHttpApi(
+async function prometheusHttpApiSupport(
   client: ClickhouseClient,
   connection: { id: string; host: string; username: string; password?: string },
-): Promise<boolean> {
+): Promise<PrometheusHttpApiSupport> {
   const key = `${connection.id}|${connection.host}`;
   const span = trace.getActiveSpan();
 
-  const kept = backendProbeCache.get<boolean>(key);
+  const kept = backendProbeCache.get<PrometheusHttpApiSupport>(key);
   if (kept !== undefined) {
     span?.setAttribute('hyperdx.promql.backend_probe', 'cached');
     return kept;
@@ -567,7 +574,7 @@ async function clickhouseServesPrometheusHttpApi(
     return served;
   } catch {
     span?.setAttribute('hyperdx.promql.backend_probe', 'inconclusive');
-    return false;
+    return NO_PROMETHEUS_HTTP_API;
   }
 }
 
@@ -586,10 +593,11 @@ export async function connectionServesPrometheusHttpApi(connection: {
   isPrometheusEndpoint?: boolean;
 }): Promise<boolean> {
   if (connection.isPrometheusEndpoint) return false;
-  return clickhouseServesPrometheusHttpApi(
+  const support = await prometheusHttpApiSupport(
     newPromqlClickhouseClient(connection),
     connection,
   );
+  return support.query;
 }
 
 type ClickhouseQueryBackend =
@@ -631,7 +639,7 @@ export async function resolveClickhouseQueryBackend({
 
   let useHttpApi: boolean;
   if (promqlBackend === 'auto') {
-    useHttpApi = await clickhouseServesPrometheusHttpApi(client, connection);
+    useHttpApi = (await prometheusHttpApiSupport(client, connection)).query;
   } else {
     span?.setAttribute('hyperdx.promql.backend_probe', 'skipped');
     useHttpApi = promqlBackend === 'clickhouse-http';
@@ -657,6 +665,36 @@ export async function resolveClickhouseQueryBackend({
   };
 }
 
+/**
+ * How to answer a label lookup (`/labels`, `/label/<name>/values`) on a
+ * ClickHouse target: through the Prometheus HTTP API, or with SQL against the
+ * tags table. The handler serves label lookups only from 26.9, three releases
+ * after it serves queries, so a source on the HTTP API reads labels with SQL
+ * on 26.6-26.8 rather than losing them. That holds for a source set to
+ * `clickhouse-http` too, so it is probed like `auto`. A source set to
+ * `table-function` reads them with SQL, with no probe.
+ */
+export async function resolveClickhouseLabelBackend({
+  connection,
+  promqlBackend = 'auto',
+}: PromqlTarget): Promise<'clickhouse-http' | 'sql'> {
+  const span = trace.getActiveSpan();
+  span?.setAttribute('hyperdx.promql.backend_setting', promqlBackend);
+
+  let labelBackend: 'clickhouse-http' | 'sql' = 'sql';
+  if (promqlBackend === 'table-function') {
+    span?.setAttribute('hyperdx.promql.backend_probe', 'skipped');
+  } else {
+    const support = await prometheusHttpApiSupport(
+      newPromqlClickhouseClient(connection),
+      connection,
+    );
+    if (support.labels) labelBackend = 'clickhouse-http';
+  }
+  span?.setAttribute('hyperdx.promql.label_backend', labelBackend);
+  return labelBackend;
+}
+
 /** How to evaluate PromQL on a target. Probes ClickHouse targets for the HTTP API. */
 export async function resolvePromqlQueryBackend(
   target: PromqlTarget,