From 2556d74543d418cc3722b025699ea05cf439b30e Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:56:13 -0500 Subject: [PATCH 1/3] fix: drop the row cap from text index value lookups --- .changeset/text-index-values-no-row-cap.md | 11 ++++++++ .../src/__tests__/metadata.test.ts | 26 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 16 +++++------- 3 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 .changeset/text-index-values-no-row-cap.md diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md new file mode 100644 index 000000000..103ed81b0 --- /dev/null +++ b/.changeset/text-index-values-no-row-cap.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/common-utils': patch +--- + +fix: show values for every key in text index value lookups + +"All values" lookups that read a text index (Map key=value indexes and text +indexes on plain columns) stopped at `max_rows_to_read` with +`read_overflow_mode = 'break'`. The index is read in sorted token order, so once +a high-cardinality key used up the budget, every key that sorts after it came +back with no values and no error. These lookups no longer apply a row cap. diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 3df0c03fc..f4c3e6804 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1403,6 +1403,32 @@ describe('Metadata', () => { expect(sql).toContain('GROUP BY column, key'); }); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads the %s text index without a row cap, even when one is configured', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + jest + .spyOn(metadata, 'getClickHouseSettings') + .mockReturnValue({ max_rows_to_read: '1000' }); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + }); + + const calls = (mockClickhouseClient.query as jest.Mock).mock.calls; + const textIndexCall = calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].clickhouse_settings).toMatchObject({ + max_rows_to_read: '0', + }); + }, + ); + it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { setupDefaultLogsSchema(); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 8226f810a..0ee8017af 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1800,12 +1800,11 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', ...this.getClickHouseSettings(), + // The dictionary is read in sorted token order, so a row cap + // with read_overflow_mode=break silently leaves every key that + // sorts after the cutoff with no values. + max_rows_to_read: '0', }, abort_signal: signal, }) @@ -1876,12 +1875,9 @@ export class Metadata { query_params: sql.params, connectionId, clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', ...this.getClickHouseSettings(), + // See getMapTextIndexKeyValues. + max_rows_to_read: '0', }, abort_signal: signal, }) From b6cb6bab97dfcad11be0b9d4f2d536dfd8fda2e0 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:05:16 -0500 Subject: [PATCH 2/3] perf: read only median-sized parts in text index value lookups --- .changeset/text-index-values-no-row-cap.md | 8 +++ packages/api/openapi.json | 8 +-- .../src/mcp/tools/sources/describeSource.ts | 20 ++++--- packages/api/src/mcp/tools/sources/schemas.ts | 12 ++-- .../src/routers/external-api/v2/sources.ts | 8 +-- .../components/DBSearchPageFilters/hooks.ts | 2 + .../SourceForm/MapKeyIndexMaxPartsFormRow.tsx | 4 +- packages/app/src/hooks/useMetadata.tsx | 48 +++++++++++----- .../src/__tests__/metadata.test.ts | 56 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 31 ++++++++-- packages/common-utils/src/types.ts | 2 +- 11 files changed, 154 insertions(+), 45 deletions(-) diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md index 103ed81b0..4799ae226 100644 --- a/.changeset/text-index-values-no-row-cap.md +++ b/.changeset/text-index-values-no-row-cap.md @@ -1,5 +1,7 @@ --- '@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch --- fix: show values for every key in text index value lookups @@ -9,3 +11,9 @@ indexes on plain columns) stopped at `max_rows_to_read` with `read_overflow_mode = 'break'`. The index is read in sorted token order, so once a high-cardinality key used up the budget, every key that sorts after it came back with no values and no error. These lookups no longer apply a row cap. + +To keep them bounded, they now read a limited set of parts like Map key +discovery: the 20 parts closest to the median size by default, or as set by the +source's "Map key index parts" (`mapKeyIndexMaxParts`) and "Map key index part +size" (`mapKeyIndexPartSelection`) settings, which now cover values as well as +keys. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 5e55d31d9..e3829f49c 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5206,7 +5206,7 @@ "mapKeyIndexMaxParts": { "type": "integer", "minimum": 1, - "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.", "nullable": true, "example": 20 }, @@ -5217,7 +5217,7 @@ "smallest", "largest" ], - "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", "nullable": true, "example": "median" }, @@ -5478,7 +5478,7 @@ "mapKeyIndexMaxParts": { "type": "integer", "minimum": 1, - "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.", "nullable": true, "example": 20 }, @@ -5489,7 +5489,7 @@ "smallest", "largest" ], - "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", "nullable": true, "example": "median" }, diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 58f623307..4811a036c 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -228,6 +228,12 @@ async function describeSourceSchema( 'metadataMaterializedViews' in source ? source.metadataMaterializedViews : undefined; + const mapKeyIndexMaxParts = + 'mapKeyIndexMaxParts' in source ? source.mapKeyIndexMaxParts : undefined; + const mapKeyIndexPartSelection = + 'mapKeyIndexPartSelection' in source + ? source.mapKeyIndexPartSelection + : undefined; const now = new Date(); const dateRange: [Date, Date] = [ @@ -306,14 +312,8 @@ async function describeSourceSchema( maxKeys: 50, connectionId, metadataMVs, - mapKeyIndexMaxParts: - 'mapKeyIndexMaxParts' in source - ? source.mapKeyIndexMaxParts - : undefined, - mapKeyIndexPartSelection: - 'mapKeyIndexPartSelection' in source - ? source.mapKeyIndexPartSelection - : undefined, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -360,6 +360,8 @@ async function describeSourceSchema( maxValuesPerKey: MAX_LC_VALUES, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -407,6 +409,8 @@ async function describeSourceSchema( maxValuesPerKey: MAX_MAP_KEY_VALUES, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 89a64bfda..b164c4f6b 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -454,17 +454,17 @@ const mcpSaveSourceSchema = z.object({ .min(1) .optional() .describe( - 'Log and trace sources. How many data parts Map attribute key discovery ' + - 'reads from a text index, chosen by mapKeyIndexPartSelection. Raise it ' + - 'if attribute keys are missing from filters; lower it if field loading ' + - 'is slow. Omit for the default of 20.', + 'Log and trace sources. How many data parts are read from text indexes ' + + 'to discover attribute keys and filter values, chosen by ' + + 'mapKeyIndexPartSelection. Raise it if keys or values are missing from ' + + 'filters; lower it if they load slowly. Omit for the default of 20.', ), mapKeyIndexPartSelection: z .enum(['median', 'smallest', 'largest']) .optional() .describe( - 'Log and trace sources. Which data parts, by row count, Map attribute ' + - 'key discovery reads from a text index: "median" (used when unset) ' + + 'Log and trace sources. Which data parts, by row count, are read from ' + + 'text indexes for attribute keys and filter values: "median" (used when unset) ' + 'balances coverage and cost, "smallest" reads the newest unmerged ' + 'parts, "largest" reads the most merged data.', ), diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 164f1c525..1d7baf10a 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -415,13 +415,13 @@ function formatExternalSource(source: SourceDocument) { * mapKeyIndexMaxParts: * type: integer * minimum: 1 - * description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20. * nullable: true * example: 20 * mapKeyIndexPartSelection: * type: string * enum: [median, smallest, largest] - * description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. * nullable: true * example: median * highlightedTraceAttributeExpressions: @@ -628,13 +628,13 @@ function formatExternalSource(source: SourceDocument) { * mapKeyIndexMaxParts: * type: integer * minimum: 1 - * description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20. * nullable: true * example: 20 * mapKeyIndexPartSelection: * type: string * enum: [median, smallest, largest] - * description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. * nullable: true * example: median * highlightedTraceAttributeExpressions: diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 54267aeaf..4b7e06e24 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -255,6 +255,8 @@ function useFacets({ tableName: tableConnection.tableName, connectionId: tableConnection.connectionId, metadataMVs: tableConnection.metadataMVs, + mapKeyIndexMaxParts: tableConnection.mapKeyIndexMaxParts, + mapKeyIndexPartSelection: tableConnection.mapKeyIndexPartSelection, keyExpressions: [sqlKey], maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT, dateRange, diff --git a/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx b/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx index aa1c52ca2..1c5a10f59 100644 --- a/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx +++ b/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx @@ -16,7 +16,7 @@ export function MapKeyIndexMaxPartsFormRow({ return ( s.id === chartConfig.source) : undefined; + const isLogOrTrace = + !!source && (isLogSource(source) || isTraceSource(source)); const metadataMVs = usableMVsOverride ?? - (source && (isLogSource(source) || isTraceSource(source)) - ? source.metadataMaterializedViews - : undefined); - return { chartConfig, source, metadataMVs }; + (isLogOrTrace ? source.metadataMaterializedViews : undefined); + return { + chartConfig, + source, + metadataMVs, + mapKeyIndexMaxParts: isLogOrTrace + ? source.mapKeyIndexMaxParts + : undefined, + mapKeyIndexPartSelection: isLogOrTrace + ? source.mapKeyIndexPartSelection + : undefined, + }; }); if (mode === 'exact') { @@ -606,18 +616,26 @@ export function useMultipleGetKeyValues( } // mode === 'all' - const streams = lookups.map(({ chartConfig, metadataMVs }) => - metadata.streamAllKeyValues({ - databaseName: chartConfig.from.databaseName, - tableName: chartConfig.from.tableName, - keyExpressions: limitedKeys, - maxValuesPerKey: limit, - connectionId: chartConfig.connection, + const streams = lookups.map( + ({ + chartConfig, metadataMVs, - dateRange: chartConfig.dateRange, - timestampValueExpression: chartConfig.timestampValueExpression, - signal, - }), + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, + }) => + metadata.streamAllKeyValues({ + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + keyExpressions: limitedKeys, + maxValuesPerKey: limit, + connectionId: chartConfig.connection, + metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, + dateRange: chartConfig.dateRange, + timestampValueExpression: chartConfig.timestampValueExpression, + signal, + }), ); for await (const keyValues of mergeStreams(streams)) { yield keyValues.map(({ key, value }) => ({ diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index f4c3e6804..cd98c25d6 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1429,6 +1429,62 @@ describe('Metadata', () => { }, ); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads only the parts nearest the median size from the %s text index', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + }); + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts: 5, + }); + + const partsLimits = (mockClickhouseClient.query as jest.Mock).mock.calls + .filter(([opts]) => opts.query.includes('mergeTreeTextIndex(')) + .map(([opts]) => { + const match = opts.query.match( + /ORDER BY abs\(rn - \(cnt \+ 1\) \/ 2\), name\s+LIMIT \{(\w+):Int32\}/, + ); + expect(match).not.toBeNull(); + return opts.query_params[match[1]]; + }); + expect(partsLimits).toEqual([20, 5]); + }, + ); + + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'orders the %s text index parts by the configured part selection', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexPartSelection: MapKeyIndexPartSelection.Largest, + }); + + const textIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].query).toMatch( + /\)\s+ORDER BY rn DESC\s+LIMIT \{\w+:Int32\}/, + ); + }, + ); + it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { setupDefaultLogsSchema(); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 0ee8017af..86cea218a 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -70,9 +70,9 @@ import { // Between 1e6 - 5e6 is a good range. export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6; const DEFAULT_MAX_KEYS = 1000; -// shortcut: keys found only in parts outside the selection can be missed; raise -// it per source (mapKeyIndexMaxParts) or pick other parts -// (mapKeyIndexPartSelection) if keys go missing. +// shortcut: keys and values found only in parts outside the selection can be +// missed; raise it per source (mapKeyIndexMaxParts) or pick other parts +// (mapKeyIndexPartSelection) if they go missing. export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. @@ -112,6 +112,9 @@ type AllKeyValuesQuery = { maxValuesPerKey?: number; connectionId: string; metadataMVs?: MetadataMaterializedViews; + /** Parts the text index paths read; see `partsOverlapFilter`. */ + mapKeyIndexMaxParts?: number; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; @@ -1744,17 +1747,21 @@ export class Metadata { tableName, connectionId, queryOptions, + maxParts, + partSelection, dateRange, timestampValueExpression, signal, }: TableConnection & { queryOptions: TextIndexMapColumnQueryOptions; + maxParts: number; + partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getMapTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1777,6 +1784,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts, + partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; const sql = chSql` @@ -1835,17 +1844,21 @@ export class Metadata { tableName, connectionId, queryOptions, + maxParts, + partSelection, dateRange, timestampValueExpression, signal, }: TableConnection & { queryOptions: TextIndexColumnQueryOptions; + maxParts: number; + partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1855,6 +1868,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts, + partSelection, }); const sql = chSql` SELECT * FROM ( @@ -2916,6 +2931,8 @@ export class Metadata { maxValuesPerKey, connectionId, metadataMVs, + mapKeyIndexMaxParts = DEFAULT_MAP_KEY_INDEX_MAX_PARTS, + mapKeyIndexPartSelection = MapKeyIndexPartSelection.Median, dateRange, timestampValueExpression, signal, @@ -3035,6 +3052,8 @@ export class Metadata { tableName, connectionId, queryOptions: mapTextIndexQueryOptions, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -3048,6 +3067,8 @@ export class Metadata { tableName, connectionId, queryOptions: nativeTextIndexQueryOptions, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 4a0348929..134e6639b 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2713,7 +2713,7 @@ export type MetadataMaterializedViews = z.infer< >; /** - * How many parts Map key discovery reads from a text index. Unset uses + * How many parts key and value discovery reads from a text index. Unset uses * DEFAULT_MAP_KEY_INDEX_MAX_PARTS. A cleared form NumberInput yields `''`. */ const MapKeyIndexMaxPartsSchema = z.preprocess( From 5bac570aae8a3c2f4c85f03e3e7c354420a5258f Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:57:12 -0500 Subject: [PATCH 3/3] fix: let "load more" read every part in the time range --- .changeset/text-index-values-no-row-cap.md | 1 + .../DBSearchPageFilters/hooks.test.tsx | 3 +++ .../components/DBSearchPageFilters/hooks.ts | 4 +-- .../src/__tests__/metadata.test.ts | 26 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 17 +++++++----- 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/.changeset/text-index-values-no-row-cap.md b/.changeset/text-index-values-no-row-cap.md index 4799ae226..3dda793ff 100644 --- a/.changeset/text-index-values-no-row-cap.md +++ b/.changeset/text-index-values-no-row-cap.md @@ -17,3 +17,4 @@ discovery: the 20 parts closest to the median size by default, or as set by the source's "Map key index parts" (`mapKeyIndexMaxParts`) and "Map key index part size" (`mapKeyIndexPartSelection`) settings, which now cover values as well as keys. +"Load more" on a single filter key still reads every part in the time range. diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index 396741201..bd089bf62 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -807,6 +807,9 @@ describe('useFetchFacets', () => { }); expect(getAllKeyValues).toHaveBeenCalledTimes(1); + expect(getAllKeyValues).toHaveBeenCalledWith( + expect.objectContaining({ mapKeyIndexMaxParts: null }), + ); expect(getKeyValuesWithMVs).not.toHaveBeenCalled(); expect(result.current.data.keyValues).toEqual([ { key: 'ServiceName', value: ['api', 'web'] }, diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 4b7e06e24..467f93800 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -255,8 +255,8 @@ function useFacets({ tableName: tableConnection.tableName, connectionId: tableConnection.connectionId, metadataMVs: tableConnection.metadataMVs, - mapKeyIndexMaxParts: tableConnection.mapKeyIndexMaxParts, - mapKeyIndexPartSelection: tableConnection.mapKeyIndexPartSelection, + // "Load more" wants every value in the time range, not a sample of parts. + mapKeyIndexMaxParts: null, keyExpressions: [sqlKey], maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT, dateRange, diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index cd98c25d6..4ed0cdd08 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1460,6 +1460,32 @@ describe('Metadata', () => { }, ); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads every part in the time range from the %s text index when the part limit is null', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts: null, + }); + + const textIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].query).toMatch( + /part_name IN \(SELECT name\s+FROM system\.parts/, + ); + expect(textIndexCall[0].query).not.toContain('row_number()'); + }, + ); + it.each([ ['map KV', "LogAttributes['requestId']"], ['native', 'TraceId'], diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 86cea218a..659584fe6 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -112,8 +112,11 @@ type AllKeyValuesQuery = { maxValuesPerKey?: number; connectionId: string; metadataMVs?: MetadataMaterializedViews; - /** Parts the text index paths read; see `partsOverlapFilter`. */ - mapKeyIndexMaxParts?: number; + /** + * Parts the text index paths read; see `partsOverlapFilter`. Unset uses the + * default; null reads every part in the time range. + */ + mapKeyIndexMaxParts?: number | null; mapKeyIndexPartSelection?: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; @@ -1754,7 +1757,8 @@ export class Metadata { signal, }: TableConnection & { queryOptions: TextIndexMapColumnQueryOptions; - maxParts: number; + /** null reads every part in the time range. */ + maxParts: number | null; partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; @@ -1784,7 +1788,7 @@ export class Metadata { tableName, dateRange, timestampValueExpression, - maxParts, + maxParts: maxParts ?? undefined, partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; @@ -1851,7 +1855,8 @@ export class Metadata { signal, }: TableConnection & { queryOptions: TextIndexColumnQueryOptions; - maxParts: number; + /** null reads every part in the time range. */ + maxParts: number | null; partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; @@ -1868,7 +1873,7 @@ export class Metadata { tableName, dateRange, timestampValueExpression, - maxParts, + maxParts: maxParts ?? undefined, partSelection, }); const sql = chSql`