fix: unknown lucene field falls through in search (re-land #2422 + #2446)

.changeset/lucene-unknown-field-falls-through.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+---
+
+fix: unknown Lucene field no longer falls through as a raw SQL identifier
+
+A Lucene field that doesn't resolve to a real column is now gated on the set of known SELECT aliases — a known alias renders as a bare identifier (ClickHouse resolves SELECT aliases in WHERE) while anything genuinely unknown renders the no-match predicate `(1 = 0)` instead of being injected as raw SQL that ClickHouse rejects with "Unknown identifier" (which previously killed both the histogram and the row table at once). As part of this, `chSqlToAliasMap` now parses selects containing ClickHouse parametric aggregate functions — the double-paren `func(params)(args)` form (e.g. `groupUniqArray(20)(col)`, `quantile(0.9)(col)`) — instead of throwing and logging "Error parsing alias map" on every value-autocomplete fetch.

packages/common-utils/src/__tests__/clickhouse.test.ts

@@ -223,6 +223,73 @@ describe('chSqlToAliasMap - alias unit test', () => {
     };
     expect(res).toEqual(aliasMap);
   });
+
+  it('Parametric aggregate functions (getKeyValues repro)', () => {
+    // node-sql-parser can't parse the double-paren `func(params)(args)` form, so
+    // these were silently dropped and spammed "Error parsing alias map". The
+    // aliases must now resolve to the full original expression.
+    const chSqlInput: ChSql = {
+      sql: 'SELECT groupUniqArray(20)(param0) AS param0, groupUniqArray(20)(param1) AS param1 FROM t',
+      params: {},
+    };
+    const res = chSqlToAliasMap(chSqlInput);
+    expect(res).toEqual({
+      param0: 'groupUniqArray(20)(param0)',
+      param1: 'groupUniqArray(20)(param1)',
+    });
+  });
+
+  it('Parametric aggregate mixed with normal and bracketed aliases', () => {
+    const chSqlInput: ChSql = {
+      sql: "SELECT Timestamp as ts, groupUniqArray(20)(ServiceName) AS svc, ResourceAttributes['x'] as rx FROM t",
+      params: {},
+    };
+    const res = chSqlToAliasMap(chSqlInput);
+    expect(res).toEqual({
+      ts: 'Timestamp',
+      svc: 'groupUniqArray(20)(ServiceName)',
+      rx: "ResourceAttributes['x']",
+    });
+  });
+
+  it('Parametric aggregate over a dotted/JSON column', () => {
+    // Parametric replacement runs before JSON replacement, so the dotted inner
+    // expression must survive intact inside the restored alias value.
+    const chSqlInput: ChSql = {
+      sql: 'SELECT groupUniqArray(20)(ResourceAttributes.service.name) AS svc FROM t',
+      params: {},
+    };
+    const res = chSqlToAliasMap(chSqlInput);
+    expect(res).toEqual({
+      svc: 'groupUniqArray(20)(ResourceAttributes.service.name)',
+    });
+  });
+
+  it('Parametric aggregate with numeric params', () => {
+    const chSqlInput: ChSql = {
+      sql: 'SELECT quantiles(0.5, 0.9)(Duration) AS q FROM t',
+      params: {},
+    };
+    const res = chSqlToAliasMap(chSqlInput);
+    expect(res).toEqual({
+      q: 'quantiles(0.5, 0.9)(Duration)',
+    });
+  });
+
+  it('Single-paren aggregates are not treated as parametric', () => {
+    // count() has a single group and must parse natively; the parametric
+    // aggregates alongside it must still resolve.
+    const chSqlInput: ChSql = {
+      sql: 'SELECT count() AS c, groupUniqArray(20)(a) AS ga, quantile(0.9)(b) AS p90 FROM t',
+      params: {},
+    };
+    const res = chSqlToAliasMap(chSqlInput);
+    expect(res).toEqual({
+      c: 'count()',
+      ga: 'groupUniqArray(20)(a)',
+      p90: 'quantile(0.9)(b)',
+    });
+  });
 });
 
 describe('computeRatio', () => {

packages/common-utils/src/__tests__/queryParser.test.ts

@@ -35,6 +35,8 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
       return { name: 'SeverityNumber', type: 'UInt8' };
     } else if (column === 'foo') {
       return { name: 'foo', type: 'String' };
+    } else if (column === 'bar') {
+      return { name: 'bar', type: 'String' };
     } else if (column === 'MaterializedExample') {
       return { name: 'MaterializedExample', type: 'String' };
     } else {
@@ -494,6 +496,49 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
     expect(actualSql).toBe(expectedSql);
   });
 
+  describe('CustomSchemaSQLSerializerV2: select alias resolution', () => {
+    // ClickHouse resolves SELECT aliases in WHERE, so a Lucene field matching a
+    // known alias must resolve to a bare identifier. A field that is neither a
+    // column nor an alias is genuinely unknown and must not be injected as raw
+    // SQL (which ClickHouse would reject as an unknown identifier).
+    const aliasSerializer = new CustomSchemaSQLSerializerV2({
+      metadata,
+      databaseName,
+      tableName,
+      connectionId,
+      implicitColumnExpression: 'Body',
+      selectAliases: new Set(['Content']),
+    });
+
+    it('resolves a known select alias as a column reference', async () => {
+      const res = await aliasSerializer.getColumnForField('Content', {});
+      expect(res).toEqual(
+        expect.objectContaining({ found: true, column: 'Content' }),
+      );
+    });
+
+    it('renders a select alias reference as a bare identifier', async () => {
+      const sql = await new SearchQueryBuilder(
+        'Content:foo',
+        aliasSerializer,
+      ).build();
+      expect(sql).toBe("((Content ILIKE '%foo%'))");
+    });
+
+    it('returns not-found for an unknown, non-alias field', async () => {
+      const res = await serializer.getColumnForField('NotAColumn', {});
+      expect(res).toEqual(expect.objectContaining({ found: false }));
+    });
+
+    it('renders an unknown, non-alias field as the no-match predicate', async () => {
+      const sql = await new SearchQueryBuilder(
+        'NotAColumn:foo',
+        serializer,
+      ).build();
+      expect(sql).toBe('((1 = 0))');
+    });
+  });
+
   describe('CustomSchemaSQLSerializerV2: implicit column falls back to bodyExpression', () => {
     // Symmetric counterpart to the one-way fallback in
     // `getEventBody` (packages/app/src/source.ts): a log source admin who