feat(mcp): read PromQL label catalog from ClickHouse's Prometheus HTTP API

.changeset/promql-label-catalog-via-clickhouse-http-api.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/api': patch
+---
+
+feat: read the MCP PromQL label catalog from ClickHouse's Prometheus HTTP API
+
+`describe_source` on a ClickHouse-backed PromQL source now lists label and metric names through the `prometheus_api_v1` handler on ClickHouse 26.9+, and falls back to SQL on older servers, servers without the handler, and sources set to `table-function`.

packages/api/src/controllers/promql.ts

@@ -263,7 +263,9 @@ export type PromqlLabelCatalog = {
 /**
  * Lists the label names and metric names a PromQL source holds in
  * [startMs, endMs], at most `limit` of each. Returns `limit + 1` entries when
- * there are more, so callers can tell the list was cut.
+ * there are more, so callers can tell the list was cut. A ClickHouse source
+ * reads them through the Prometheus HTTP API where resolveClickhouseLabelBackend
+ * allows it, and with SQL otherwise.
  */
 export async function getPromqlLabelCatalog({
   teamId,
@@ -277,6 +279,7 @@ export async function getPromqlLabelCatalog({
   source: {
     connection: { toString(): string };
     from: { databaseName: string; tableName: string };
+    promqlBackend?: PromqlSourceBackend;
   };
   startMs: number;
   endMs: number;
@@ -292,29 +295,51 @@ export async function getPromqlLabelCatalog({
     throw new Error('Connection not found');
   }
 
+  const params = {
+    start: String(startMs / 1000),
+    end: String(endMs / 1000),
+    limit: String(limit + 1),
+  };
+  const fetchCatalog = (fetchLabels: (path: string) => Promise<string[]>) =>
+    Promise.all([
+      fetchLabels('/labels'),
+      fetchLabels('/label/__name__/values'),
+    ]);
+
   if (connection.isPrometheusEndpoint) {
-    const params = {
-      start: String(startMs / 1000),
-      end: String(endMs / 1000),
-      limit: String(limit + 1),
-    };
-    const [labelNames, metricNames] = await Promise.all([
+    const [labelNames, metricNames] = await fetchCatalog(path =>
       fetchPrometheusData(
         connection.host,
-        '/api/v1/labels',
+        `/api/v1${path}`,
         params,
         labelListSchema,
         signal,
       ),
-      fetchPrometheusData(
-        connection.host,
-        '/api/v1/label/__name__/values',
-        params,
+    );
+    return { backend: 'prometheus', labelNames, metricNames };
+  }
+
+  const { databaseName, tableName } = source.from;
+  const labelBackend = await resolveClickhouseLabelBackend({
+    connection,
+    databaseName,
+    tableName,
+    promqlBackend: source.promqlBackend,
+  });
+  if (labelBackend === 'clickhouse-http') {
+    const [labelNames, metricNames] = await fetchCatalog(path => {
+      const { upstream, headers } = clickhousePrometheusRequest(connection);
+      return fetchPrometheusData(
+        upstream,
+        `${CLICKHOUSE_PROMETHEUS_API_PREFIX}${path}`,
+        { ...params, database: databaseName, table: tableName },
         labelListSchema,
         signal,
-      ),
-    ]);
-    return { backend: 'prometheus', labelNames, metricNames };
+        headers,
+        CLICKHOUSE_HTTP_API_UNAVAILABLE_MESSAGE,
+      );
+    });
+    return { backend: 'clickhouse', labelNames, metricNames };
   }
 
   const args = {
@@ -325,8 +350,8 @@ export async function getPromqlLabelCatalog({
       requestTimeout: PROMETHEUS_FETCH_TIMEOUT_MS,
     }),
     connectionId: connection.id,
-    databaseName: source.from.databaseName,
-    tableName: source.from.tableName,
+    databaseName,
+    tableName,
     startMs,
     endMs,
     limit: limit + 1,

packages/api/src/mcp/__tests__/promqlSources.int.test.ts

@@ -1,8 +1,13 @@
+import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { SourceKind } from '@hyperdx/common-utils/dist/types';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 
 import * as config from '@/config';
-import { PROMETHEUS_MAX_RESPONSE_BYTES } from '@/controllers/promql';
+import {
+  clearBackendProbeCache,
+  CLICKHOUSE_PROMETHEUS_API_PREFIX,
+  PROMETHEUS_MAX_RESPONSE_BYTES,
+} from '@/controllers/promql';
 import {
   DEFAULT_DATABASE,
   dropTimeSeriesTable,
@@ -16,6 +21,7 @@ import { Source } from '@/models/source';
 import { callTool, createTestClient, getFirstText } from './mcpTestUtils';
 
 const mockFetch = jest.mocked(global.fetch);
+const { realFetch } = globalThis as unknown as { realFetch: typeof fetch };
 
 const TABLE = 'mcp_promql_sources';
 
@@ -88,19 +94,24 @@ describe('MCP PromQL sources', () => {
       username: config.CLICKHOUSE_USER,
       password: config.CLICKHOUSE_PASSWORD,
     });
+    // Pinned so these tests read labels with SQL on any server version; the
+    // HTTP API path is covered on its own below.
     promqlSource = await Source.create({
       kind: SourceKind.Promql,
       team: team._id,
       connection: connection._id,
       name: 'TimeSeries metrics',
       from: { databaseName: DEFAULT_DATABASE, tableName: TABLE },
       timestampValueExpression: 'timestamp',
+      promqlBackend: 'table-function',
     });
   });
 
   afterEach(async () => {
     await client?.close();
     await server.clearDBs();
+    clearBackendProbeCache();
+    mockFetch.mockReset();
   });
 
   afterAll(async () => {
@@ -250,6 +261,128 @@ describe('MCP PromQL sources', () => {
       expect(output.error).toContain('returned more than 8 MB');
     });
 
+    describe("through ClickHouse's prometheus_api_v1 handler", () => {
+      let autoSource: any;
+      let versionSpy: jest.SpyInstance | undefined;
+
+      const stubServerVersion = (version: [number, number, number, number]) => {
+        versionSpy = jest
+          .spyOn(Metadata.prototype, 'getServerVersion')
+          .mockResolvedValue(version);
+      };
+
+      const describeAutoSource = async () => {
+        const result = await callTool(client, 'clickstack_describe_source', {
+          sourceId: autoSource._id.toString(),
+        });
+        expect(result.isError).toBeFalsy();
+        return JSON.parse(getFirstText(result));
+      };
+
+      const labelLookupUrls = () =>
+        mockFetch.mock.calls
+          .map(([input]) => new URL(String(input)))
+          .filter(
+            url =>
+              url.pathname !==
+              `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/format_query`,
+          );
+
+      beforeEach(async () => {
+        autoSource = await Source.create({
+          kind: SourceKind.Promql,
+          team: team._id,
+          connection: promqlSource.connection,
+          name: 'TimeSeries metrics (auto)',
+          from: { databaseName: DEFAULT_DATABASE, tableName: TABLE },
+          timestampValueExpression: 'timestamp',
+        });
+      });
+
+      afterEach(() => {
+        versionSpy?.mockRestore();
+        versionSpy = undefined;
+      });
+
+      it("sends the label lookups with the source's table and the connection's credentials", async () => {
+        stubServerVersion([26, 10, 0, 0]);
+        mockFetch.mockImplementation(async input => {
+          const { pathname } = new URL(String(input));
+          if (pathname.endsWith('/format_query')) {
+            return prometheusResponse('up');
+          }
+          return prometheusResponse(
+            pathname.endsWith('/label/__name__/values')
+              ? ['up']
+              : ['__name__', 'job'],
+          );
+        });
+
+        const output = await describeAutoSource();
+
+        expect(output.source).toMatchObject({
+          backend: 'clickhouse',
+          table: `${DEFAULT_DATABASE}.${TABLE}`,
+        });
+        expect(output.labelNames).toEqual(['__name__', 'job']);
+        expect(output.metricNames).toEqual(['up']);
+        const urls = labelLookupUrls();
+        expect(urls.map(u => u.pathname).sort()).toEqual([
+          `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/label/__name__/values`,
+          `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/labels`,
+        ]);
+        for (const url of urls) {
+          expect(url.searchParams.getAll('database')).toEqual([
+            DEFAULT_DATABASE,
+          ]);
+          expect(url.searchParams.getAll('table')).toEqual([TABLE]);
+          expect(url.searchParams.get('start')).toBeTruthy();
+          expect(url.searchParams.get('end')).toBeTruthy();
+          expect(url.searchParams.get('limit')).toBe('201');
+        }
+        for (const [input, init] of mockFetch.mock.calls) {
+          if (new URL(String(input)).pathname.endsWith('/format_query')) {
+            continue;
+          }
+          expect(init?.headers).toMatchObject({
+            'X-ClickHouse-User': config.CLICKHOUSE_USER,
+            'X-ClickHouse-Key': config.CLICKHOUSE_PASSWORD,
+            'X-ClickHouse-Query-Id': expect.stringMatching(/^hdx-mcp-/),
+          });
+        }
+      });
+
+      // 26.8's handler answers label lookups with a 400 ("not implemented").
+      it('reads labels with SQL on ClickHouse 26.8, whose handler serves only queries', async () => {
+        stubServerVersion([26, 8, 36, 3]);
+        mockFetch.mockImplementation(async () => prometheusResponse('up'));
+
+        const output = await describeAutoSource();
+
+        expect(output.labelNames).toEqual(['__name__', 'instance', 'job']);
+        expect(output.metricNames).toEqual(['http_requests_total', 'up']);
+        expect(labelLookupUrls()).toEqual([]);
+      });
+
+      // Needs ClickHouse 26.9+, whose handler serves label lookups.
+      it('lists labels from the CI ClickHouse handler', async () => {
+        mockFetch.mockImplementation(realFetch);
+
+        const output = await describeAutoSource();
+
+        expect(output.labelNames).toEqual(['__name__', 'instance', 'job']);
+        expect(output.metricNames).toEqual(['http_requests_total', 'up']);
+        expect(
+          labelLookupUrls()
+            .map(u => u.pathname)
+            .sort(),
+        ).toEqual([
+          `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/label/__name__/values`,
+          `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/labels`,
+        ]);
+      });
+    });
+
     it('keeps promqlBackend through a describe -> rename -> save round trip', async () => {
       const pinned = await Source.create({
         kind: SourceKind.Promql,