perf: show the most frequent text index values, in batches of 10 keys

.changeset/text-index-dictionary-block-size.md

@@ -0,0 +1,18 @@
+---
+'@hyperdx/otel-collector': patch
+---
+
+perf: use 8,192-token dictionary blocks for every text index in the seed
+schemas
+
+Every text index on `otel_logs`, `otel_traces` and `hyperdx_sessions` now sets
+`dictionary_block_size = 8192` (the default is 512). Each part's sparse index
+is 16x smaller. Reading filter values with `mergeTreeTextIndex()` loads that
+index for every part on every read, and on a 537M-row table 4,096-token blocks
+already made those reads about 10x cheaper per part, made searches for a rare
+attribute value 3-7x faster, and made searches for a common value 10-30%
+faster. The setting is a per-index argument rather than the table setting
+`text_index_dictionary_block_size`, which ClickHouse only accepts from 26.6.
+
+Only new installs get the new schema. Existing tables keep their indexes until
+the indexes are cleared and materialized again.

.changeset/topk-text-index-values.md

@@ -0,0 +1,26 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+perf: read large-block Map text index values per key, most frequent first
+
+"All values" lookups on Map key=value text indexes with large dictionary blocks
+(4096 or more, e.g. `dictionary_block_size = 8192` as the seed schemas now use)
+now read each key separately, in parallel queries of at most 10 keys, and stop
+after 20,000 tokens per key, so a key with a value per row (request IDs, UUIDs)
+can't hold up the others. Values come from `topKWeighted(limit)` weighted by
+each token's row count, so a capped list holds the most common values, most
+common first. These reads no longer apply `max_rows_to_read`: the index is read
+in sorted token order, so the cap left every key that sorts after a
+high-cardinality key with no values.
+
+Indexes with smaller blocks, including ClickHouse's default of 512, keep the
+previous query shape (one read for all of a query's keys, the alphabetically
+first values, and the `max_rows_to_read` cap), now in parallel queries of at
+most 10 keys, so the cap's cut-off affects at most 9 other keys. Every read loads
+each part's sparse index, which is large with small blocks, so one read per key
+would multiply that cost. The block size comes from the index definition, then
+the table's `text_index_dictionary_block_size`, then the server's.
+
+Text indexes on plain columns now return the most frequent values with
+`topKWeighted`, in parallel queries of at most 10 columns, without the row cap.

docker/clickhouse/local/init-db-e2e.sh

@@ -40,14 +40,14 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs
   \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)),
   \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)),
   \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)),
-  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
+  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192)
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
@@ -118,10 +118,10 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces
     INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1,
     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
     INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
 )

docker/otel-collector/schema/seed/00002_otel_logs.sql

@@ -29,15 +29,18 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))),
   `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))),
   `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))),
-  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
-  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'),
-  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'),
-  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
+  -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
+  -- part's sparse index is 16x smaller, which makes reading filter values
+  -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
+  INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha', dictionary_block_size = 8192)
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)

docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql

@@ -20,8 +20,11 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
   `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)),
   `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)),
   INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'),
-  INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
+  -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
+  -- part's sparse index is 16x smaller, which makes reading filter values
+  -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
+  INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+  INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,

docker/otel-collector/schema/seed/00005_otel_traces.sql

@@ -27,14 +27,17 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
     `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(3)),
     `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))),
     `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))),
-    INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'),
-    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'),
-    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'),
-    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'),
-    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'),
-    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'),
+    -- Text indexes use 8,192-token dictionary blocks (the default is 512). Each
+    -- part's sparse index is 16x smaller, which makes reading filter values
+    -- with mergeTreeTextIndex() and looking up tokens much cheaper on large parts.
+    INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
+    INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192),
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
-    INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName))
+    INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName), dictionary_block_size = 8192)
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)

packages/app/src/components/DBRowTable.tsx

@@ -29,7 +29,10 @@ import {
   isJSDataTypeJSONStringifiable,
   JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
+import {
+  getEngineSetting,
+  splitAndTrimWithBracket,
+} from '@hyperdx/common-utils/dist/core/utils';
 import {
   DENOISE_NOISE_THRESHOLD,
   DENOISE_SAMPLE_SIZE,
@@ -1545,14 +1548,6 @@ const BLOCK_COLUMN_SETTINGS = [
   { column: '_block_offset', setting: 'enable_block_offset_column' },
 ];
 
-// engine_full ends with the SETTINGS clause.
-function getEngineSetting(engineFull: string, name: string) {
-  const settingsClause = engineFull.match(/ SETTINGS (.*)$/)?.[1] ?? '';
-  return splitAndTrimWithBracket(settingsClause)
-    .map(setting => setting.split(' = '))
-    .find(([key]) => key === name)?.[1];
-}
-
 function omitsBlockColumnSetting(engineFull: string) {
   return BLOCK_COLUMN_SETTINGS.some(
     ({ setting }) => getEngineSetting(engineFull, setting) == null,

packages/common-utils/src/__tests__/kvItemsRewrite.int.test.ts

@@ -70,7 +70,7 @@ describe('KV items rewrite integration', () => {
         LogAttributeItems Array(String) MATERIALIZED
           arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
           CODEC(ZSTD(1)),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
       )
       ENGINE = MergeTree
       ORDER BY (Timestamp)`,

packages/common-utils/src/__tests__/metadata.int.test.ts

@@ -697,7 +697,7 @@ describe('Metadata Integration Tests', () => {
             ResourceAttributeItems Array(String) MATERIALIZED
               arrayMap(x -> concat(x.1, '=', x.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))
               CODEC(ZSTD(1)),
-            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
@@ -712,7 +712,7 @@ describe('Metadata Integration Tests', () => {
             LogAttributeItems Array(String) MATERIALIZED
               arrayMap((x) -> concat(x.1, '=', x.2), LogAttributes::Array(Tuple(String, String)))
               CODEC(ZSTD(1)),
-            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1
+            INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1
           )
           ENGINE = MergeTree()
           ORDER BY (Timestamp)
@@ -907,6 +907,41 @@ describe('Metadata Integration Tests', () => {
       );
     });
 
+    it('keeps the most frequent text index values when maxValuesPerKey truncates the list', async () => {
+      if (!textIndexSupported) {
+        console.warn(
+          'Skipping: ClickHouse < 26.3 does not support mergeTreeTextIndex()',
+        );
+        return;
+      }
+      // The index dictionary is sorted, so 'a-rare' is read first.
+      const freqKey = `freq.${tag}`;
+      const timestamp = testTime.toISOString().replace('T', ' ').slice(0, 23);
+      await client.command({
+        query: `INSERT INTO default.otel_logs (Timestamp, ResourceAttributes)
+          SELECT '${timestamp}', map('${freqKey}', if(number = 0, 'a-rare', 'z-common'))
+          FROM numbers(4)`,
+      });
+
+      const result = await metadata.getAllKeyValues({
+        ...commonArgs,
+        keyExpressions: [
+          `ResourceAttributes['${freqKey}']`,
+          `ResourceAttributes['${mapKey}']`,
+        ],
+        maxValuesPerKey: 1,
+      });
+
+      // UNION ALL branches can arrive in any order.
+      expect(result).toHaveLength(2);
+      expect(result).toEqual(
+        expect.arrayContaining([
+          { key: `ResourceAttributes['${freqKey}']`, value: ['z-common'] },
+          { key: `ResourceAttributes['${mapKey}']`, value: [podNameA] },
+        ]),
+      );
+    });
+
     it('routes TraceId through getTextIndexKeyValues', async () => {
       if (!textIndexSupported) {
         console.warn(
@@ -1575,14 +1610,14 @@ describe('Metadata Integration Tests', () => {
     {
       path: 'key',
       tableName: 'test_map_keys_key_text_index_order',
-      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 1`,
+      indexDefinition: `INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
     {
       path: 'kv',
       tableName: 'test_map_keys_kv_text_index_order',
       indexDefinition: `LogAttributeItems Array(String) MATERIALIZED
           arrayMap(x -> concat(x.1, '=', x.2), CAST(LogAttributes, 'Array(Tuple(String, String))')),
-        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array') GRANULARITY 1`,
+        INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array', dictionary_block_size = 8192) GRANULARITY 1`,
     },
   ])(
     'getMapKeys - $path text index ordering',