fix(app): address JSON array elements by index in row-panel filters

.changeset/json-array-paths.md

@@ -0,0 +1,9 @@
+---
+'@hyperdx/app': patch
+---
+
+fix: address JSON array elements by index in row-panel filters
+
+"Add to Filters" on a value inside an array in a native JSON column now uses
+1-based array subscripts, for example `events[1]`, instead of quoted keys such
+as `events.0`, so the filter matches rows instead of silently returning none.

.changeset/json-filter-keys-reescape.md

@@ -0,0 +1,9 @@
+---
+'@hyperdx/app': patch
+---
+
+fix: keep JSON-column filter keys in JSON form when another filter is added
+
+Adding a second filter no longer rewrites an earlier filter on a key inside a
+native JSON column to `col['key']`, which ClickHouse rejects with
+`Code 43: First argument for function 'arrayElement' must be array, got 'JSON'`.

packages/app/src/DBSearchPage.tsx

@@ -1430,6 +1430,19 @@ function DBSearchPageContent() {
         : new Set<string>(),
     [inputSourceColumns],
   );
+  // Native JSON columns, so dotted filter keys under them are written as JSON
+  // access (`toString(`col`.`a`)`) and not as Map brackets (`col['a']`).
+  const jsonColumns = useMemo(
+    () =>
+      inputSourceColumns
+        ? new Set(
+            inputSourceColumns
+              .filter(c => c.type === 'JSON' || c.type.startsWith('JSON('))
+              .map(c => c.name),
+          )
+        : new Set<string>(),
+    [inputSourceColumns],
+  );
 
   const watchedSource = useWatch({
     control,
@@ -1468,6 +1481,7 @@ function DBSearchPageContent() {
     onFilterChange: handleSetFilters,
     dateTimeColumns,
     knownColumns,
+    jsonColumns,
   });
 
   useEffect(() => {

packages/app/src/__tests__/searchFilters.test.ts

@@ -990,6 +990,62 @@ describe('searchFilters', () => {
       ]);
     });
 
+    it('keeps an existing JSON-column filter in JSON form when another filter is added', () => {
+      onFilterChange.mockClear();
+      const { result } = renderHook(() =>
+        useSearchPageFilterState({
+          // An earlier chip on a key inside the JSON column `message`, as
+          // persisted in the URL.
+          searchQuery: [
+            {
+              type: 'sql',
+              condition: "toString(`message`.`email`) IN ('user@example.test')",
+            },
+          ],
+          onFilterChange,
+          knownColumns: new Set(['message', 'source_name']),
+          jsonColumns: new Set(['message']),
+        }),
+      );
+
+      act(() => {
+        result.current.setFilterValue('source_name', 'EMAIL_SERVICE');
+      });
+
+      const conditions = (
+        onFilterChange.mock.lastCall![0] as ConditionFilter[]
+      ).map(f => f.condition);
+      // Re-persisting must not rewrite the existing key into Map bracket form
+      // (`message['email']`), which ClickHouse rejects for a JSON column.
+      expect(conditions).toContain(
+        "toString(`message`.`email`) IN ('user@example.test')",
+      );
+      expect(conditions.some(c => c.includes("['"))).toBe(false);
+    });
+
+    it('keeps an existing JSON array-path filter intact when another filter is added', () => {
+      onFilterChange.mockClear();
+      const arrayPathCondition =
+        "toString(`message`.`events`[1].`detail`.`parameters`[2].`value`) IN ('EMAIL_SERVICE')";
+      const { result } = renderHook(() =>
+        useSearchPageFilterState({
+          searchQuery: [{ type: 'sql', condition: arrayPathCondition }],
+          onFilterChange,
+          knownColumns: new Set(['message', 'source_name']),
+          jsonColumns: new Set(['message']),
+        }),
+      );
+
+      act(() => {
+        result.current.setFilterValue('source_name', 'EMAIL_SERVICE');
+      });
+
+      const conditions = (
+        onFilterChange.mock.lastCall![0] as ConditionFilter[]
+      ).map(f => f.condition);
+      expect(conditions).toContain(arrayPathCondition);
+    });
+
     it('setOnlyFilters applies all filters in a single onFilterChange emit', () => {
       onFilterChange.mockClear();
       const { result } = renderHook(() =>

packages/app/src/__tests__/utils.test.ts

@@ -1373,6 +1373,40 @@ describe('mergePath', () => {
         'BodyJson.`service`.`name`',
       );
     });
+
+    // Without array positions a numeric segment is a JSON object key, as
+    // before: facet keys come from the JSON schema, where `1` is a real key.
+    it('keeps a numeric segment as a quoted key when no array positions are given', () => {
+      expect(mergePath(['Body', 'events', '0', 'id'], ['Body'])).toBe(
+        'Body.`events`.`0`.`id`',
+      );
+    });
+
+    // The row viewer knows which segments index into an array. A JSON array
+    // element is addressed with a 1-based subscript; the quoted key
+    // (`.`0``) matches nothing, so the filter silently returns no rows.
+    it('emits a 1-based subscript for segments marked as array positions', () => {
+      expect(
+        mergePath(
+          ['message', 'events', '0', 'detail', 'parameters', '1', 'value'],
+          ['message'],
+          [],
+          new Set([2, 5]),
+        ),
+      ).toBe('message.`events`[1].`detail`.`parameters`[2].`value`');
+    });
+
+    it('chains subscripts for nested arrays', () => {
+      expect(
+        mergePath(['m', 'grid', '0', '1', 'x'], ['m'], [], new Set([2, 3])),
+      ).toBe('m.`grid`[1][2].`x`');
+    });
+
+    it('keeps a numeric object key quoted when it is not marked as an array position', () => {
+      expect(mergePath(['m', 'counts', '200', 'n'], ['m'], [], new Set())).toBe(
+        'm.`counts`.`200`.`n`',
+      );
+    });
   });
 
   describe('Map column (HDX-4369)', () => {

packages/app/src/components/DBRowJsonViewer.test.tsx

@@ -3,7 +3,11 @@ import { fireEvent, screen, within } from '@testing-library/react';
 
 import { useMaterializedAliasColumnsOption } from '@/hooks/useMaterializedAliasColumnsOption';
 
-import { buildJSONExtractQuery, DBRowJsonViewer } from './DBRowJsonViewer';
+import {
+  buildJSONExtractQuery,
+  DBRowJsonViewer,
+  getArrayIndexPositions,
+} from './DBRowJsonViewer';
 import { RowSidePanelContext } from './DBRowSidePanel';
 
 // Mock Next.js router
@@ -446,3 +450,42 @@ describe('DBRowJsonViewer', () => {
     });
   });
 });
+
+describe('getArrayIndexPositions', () => {
+  const row = {
+    message: {
+      events: [
+        { detail: { parameters: [{ name: 'a' }, { name: 'b', value: 'x' }] } },
+      ],
+      counts: { '200': { n: 1 } },
+    },
+  };
+
+  it('marks the path positions whose parent is an array', () => {
+    expect(
+      Array.from(
+        getArrayIndexPositions(row, [
+          'message',
+          'events',
+          '0',
+          'detail',
+          'parameters',
+          '1',
+          'value',
+        ]),
+      ),
+    ).toEqual([2, 5]);
+  });
+
+  it('does not mark numeric keys of a plain object', () => {
+    expect(
+      Array.from(
+        getArrayIndexPositions(row, ['message', 'counts', '200', 'n']),
+      ),
+    ).toEqual([]);
+  });
+
+  it('returns an empty set for a missing path', () => {
+    expect(getArrayIndexPositions(row, ['nope', '0']).size).toBe(0);
+  });
+});

packages/app/src/components/DBRowJsonViewer.tsx

@@ -422,6 +422,23 @@ function HyperJsonMenu({
   );
 }
 
+// Positions in `keyPath` whose parent value in `rowData` is an array, i.e. the
+// segments that are array indexes and not object keys.
+export function getArrayIndexPositions(
+  rowData: unknown,
+  keyPath: string[],
+): Set<number> {
+  const positions = new Set<number>();
+  let node: any = rowData;
+  keyPath.forEach((segment, i) => {
+    if (Array.isArray(node)) {
+      positions.add(i);
+    }
+    node = node?.[segment];
+  });
+  return positions;
+}
+
 export function DBRowJsonViewer({
   data,
   jsonColumns,
@@ -495,7 +512,12 @@ export function DBRowJsonViewer({
   const getLineActions = useCallback<GetLineActions>(
     ({ keyPath, value, isInParsedJson, parsedJsonRootPath }) => {
       const actions: LineAction[] = [];
-      const fieldPath = mergePath(keyPath, jsonColumns, mapColumns);
+      const fieldPath = mergePath(
+        keyPath,
+        jsonColumns,
+        mapColumns,
+        getArrayIndexPositions(rowData, keyPath),
+      );
       const isJsonColumn =
         keyPath.length > 0 && jsonColumns?.includes(keyPath[0]);
 

packages/app/src/components/DBSearchPageFilters.tsx

@@ -1259,6 +1259,11 @@ const DBSearchPageFiltersComponent = ({
     () => (columns ? new Set(columns.map(c => c.name)) : new Set<string>()),
     [columns],
   );
+  // Same JSON columns as `jsonColumns` above, as a Set for key escaping.
+  const jsonColumnSet = useMemo(
+    () => new Set(jsonColumns ?? []),
+    [jsonColumns],
+  );
 
   const [showMoreFields, setShowMoreFields] = useState(false);
   const {
@@ -1581,6 +1586,7 @@ const DBSearchPageFiltersComponent = ({
                 sqlKey: toQuotedClickHouseKeyExpression(
                   child.key,
                   knownColumns,
+                  jsonColumnSet,
                 ),
               }))}
               selectedValues={group.children.reduce((acc, child) => {
@@ -1647,6 +1653,7 @@ const DBSearchPageFiltersComponent = ({
             const facetSqlKey = toQuotedClickHouseKeyExpression(
               facet.key,
               knownColumns,
+              jsonColumnSet,
             );
             return (
               <FilterGroup
@@ -1737,6 +1744,7 @@ const DBSearchPageFiltersComponent = ({
       setFilterRange,
       tableMetadata,
       knownColumns,
+      jsonColumnSet,
       extraFacetKeys,
     ],
   );

packages/app/src/components/DBSearchPageFilters/utils.test.ts

@@ -435,3 +435,74 @@ describe('toQuotedClickHouseKeyExpression', () => {
     });
   });
 });
+
+describe('toQuotedClickHouseKeyExpression with JSON columns', () => {
+  const knownColumns = new Set(['message', 'LogAttributes']);
+  const jsonColumns = new Set(['message']);
+
+  it('keeps a dotted key under a JSON column in JSON dot access, never Map brackets', () => {
+    // `message` is a native JSON column; `message['email']` makes ClickHouse
+    // fail with "First argument for function 'arrayElement' must be array".
+    expect(
+      toQuotedClickHouseKeyExpression(
+        'message.email',
+        knownColumns,
+        jsonColumns,
+      ),
+    ).toBe('toString(`message`.`email`)');
+  });
+
+  it('splits a nested JSON path into one quoted segment per level', () => {
+    expect(
+      toQuotedClickHouseKeyExpression(
+        'message.owner.profile.contact.email',
+        knownColumns,
+        jsonColumns,
+      ),
+    ).toBe('toString(`message`.`owner`.`profile`.`contact`.`email`)');
+  });
+
+  // The row panel writes a filter on a value inside a JSON array as
+  // ``col.`events`[1].`id` ``. After the state cleans the backticks, the
+  // subscript must survive re-escaping as a subscript, not as part of a name.
+  it('keeps array subscripts in a JSON path as subscripts', () => {
+    expect(
+      toQuotedClickHouseKeyExpression(
+        'message.events[1].detail.parameters[2].value',
+        knownColumns,
+        jsonColumns,
+      ),
+    ).toBe('toString(`message`.`events`[1].`detail`.`parameters`[2].`value`)');
+  });
+
+  it('keeps chained array subscripts', () => {
+    expect(
+      toQuotedClickHouseKeyExpression(
+        'message.grid[1][2].x',
+        knownColumns,
+        jsonColumns,
+      ),
+    ).toBe('toString(`message`.`grid`[1][2].`x`)');
+  });
+
+  it('is idempotent on a key already in JSON form', () => {
+    const once = toQuotedClickHouseKeyExpression(
+      'message.email',
+      knownColumns,
+      jsonColumns,
+    );
+    expect(
+      toQuotedClickHouseKeyExpression(once, knownColumns, jsonColumns),
+    ).toBe(once);
+  });
+
+  it('still uses bracket access for a Map column', () => {
+    expect(
+      toQuotedClickHouseKeyExpression(
+        'LogAttributes.host',
+        knownColumns,
+        jsonColumns,
+      ),
+    ).toBe("LogAttributes['host']");
+  });
+});