From 375047b0d656374d520c8712aa295ff7d9a855bf Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:43:44 -0500 Subject: [PATCH] feat: skip filesystem cache writes for long time ranges --- .changeset/smart-filesystem-cache.md | 7 +++ packages/api/src/models/source.ts | 5 ++ .../Sources/SourceForm/SourceForm.tsx | 16 ++++++ .../Sources/SourceForm/constants.ts | 9 ++++ .../hooks/__tests__/useChartConfig.test.tsx | 40 ++++++++++++++ packages/app/src/hooks/useChartConfig.tsx | 4 +- .../app/src/hooks/useOffsetPaginatedQuery.tsx | 4 +- .../common-utils/src/__tests__/utils.test.ts | 52 +++++++++++++++++++ packages/common-utils/src/core/utils.ts | 33 ++++++++++++ packages/common-utils/src/types.ts | 9 ++++ 10 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 .changeset/smart-filesystem-cache.md diff --git a/.changeset/smart-filesystem-cache.md b/.changeset/smart-filesystem-cache.md new file mode 100644 index 0000000000..c96fc4a9e5 --- /dev/null +++ b/.changeset/smart-filesystem-cache.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': minor +--- + +feat: New per-source setting "Skip filesystem cache writes". Charts and searches over a time range longer than the chosen threshold set `read_from_filesystem_cache_if_exists_otherwise_bypass_cache = 1`, so long scans don't evict recently cached data. Off by default. diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 19902a41a0..759a299a10 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -111,6 +111,11 @@ const sourceBaseSchema = new Schema( message: '{PATH} exceeds the limit of 10', }, }, + // Mirrors the '' -> undefined preprocess in BaseSourceSchema. + filesystemCacheReadOnlyMinRange: { + type: String, + set: (v: string | null | undefined) => (v === '' ? undefined : v), + }, }, { discriminatorKey: 'kind', diff --git a/packages/app/src/components/Sources/SourceForm/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm/SourceForm.tsx index f3d9f5e792..2eccaa2b29 100644 --- a/packages/app/src/components/Sources/SourceForm/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/SourceForm.tsx @@ -40,6 +40,7 @@ import { AutocompleteControlled, InputControlled, } from '@/components/InputControlled'; +import SelectControlled from '@/components/SelectControlled'; import { IS_METRICS_ENABLED, IS_PROMQL_ENABLED, @@ -62,6 +63,7 @@ import { import { DEFAULT_DATABASE, + FILESYSTEM_CACHE_READ_ONLY_MIN_RANGE_OPTIONS, PROMETHEUS_PLACEHOLDER, SOURCE_KIND_LABELS, } from './constants'; @@ -746,6 +748,20 @@ export function TableSourceForm({ Add Setting + + + option.value !== '1 second'), ]; +export const FILESYSTEM_CACHE_READ_ONLY_MIN_RANGE_OPTIONS = [ + { value: '', label: 'Never' }, + { value: '6 hour', label: 'Longer than 6 hours' }, + { value: '1 day', label: 'Longer than 1 day' }, + { value: '2 day', label: 'Longer than 2 days' }, + { value: '7 day', label: 'Longer than 7 days' }, + { value: '30 day', label: 'Longer than 30 days' }, +]; + export const USE_TEXT_INDEX_OPTIONS = [ { value: UseTextIndex.Auto, diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx index a20788a176..94a34e3d63 100644 --- a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -25,6 +25,7 @@ import { useQueriedChartConfig, } from '@/hooks/useChartConfig'; import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanation'; +import * as sourceModule from '@/source'; // Mock DEFAULT_TIME_WINDOWS_SECONDS to remove the 15m window jest.mock('@/utils/searchWindows', () => { @@ -1672,6 +1673,45 @@ describe('useChartConfig', () => { expect(result.current.isPending).toBe(false); }); + it('applies the filesystem cache setting to every chunk based on the full range', async () => { + // 3 days, split into windows that are each shorter than the 2-day threshold. + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-04 00:00:00Z'), + ], + granularity: '1 hour', + }); + mockClickhouseClient.queryChartConfig.mockResolvedValue( + createMockQueryResponse([]), + ); + const useSourceSpy = jest + .spyOn(sourceModule, 'useSource') + .mockReturnValue({ + data: { filesystemCacheReadOnlyMinRange: '2 day' }, + isLoading: false, + } as any); + + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { wrapper }, + ); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + useSourceSpy.mockRestore(); + + const calls = mockClickhouseClient.queryChartConfig.mock.calls; + expect(calls.length).toBeGreaterThan(1); + for (const [{ querySettings }] of calls) { + expect(querySettings).toEqual([ + { + setting: + 'read_from_filesystem_cache_if_exists_otherwise_bypass_cache', + value: '1', + }, + ]); + } + }); + it('pins the series-limit ranking to the newest window on each chunk', async () => { const config = createMockChartConfig({ dateRange: [ diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index ed367f0c27..6f3ca06157 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -19,6 +19,7 @@ import { import { convertDateRangeToGranularityString, convertGranularityToSeconds, + getSourceQuerySettings, hasPositiveSeriesLimit, } from '@hyperdx/common-utils/dist/core/utils'; import { @@ -449,7 +450,8 @@ export function useQueriedChartConfig( enableParallelQueries: options?.enableParallelQueries, metadata, querySettings: mergeQuerySettings( - source?.querySettings, + // The full range, before fetchDataInChunks splits it into windows. + getSourceQuerySettings(source, config.dateRange), options?.additionalQuerySettings, ), }); diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 9492d435d0..655bb032a1 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -15,6 +15,7 @@ import { import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { + getSourceQuerySettings, isFirstOrderByAscending, isTimestampExpressionInFirstOrderBy, MAX_TABLE_ROWS, @@ -257,7 +258,8 @@ const queryFn: QueryFunction = async ({ const query = await renderChartConfig( windowedConfig, metadata, - source?.querySettings, + // The full range, not the page's time window. + getSourceQuerySettings(source, config.dateRange), ); // Create abort signal from timeout if provided diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 327585152b..ae05222519 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -15,6 +15,7 @@ import { getDistributedTableArgs, getFirstOrderingItem, getPercentileAggFnLevel, + getSourceQuerySettings, hasNonEmptyOrderBy, hasPositiveSeriesLimit, isFirstOrderByAscending, @@ -3099,6 +3100,57 @@ describe('utils', () => { ).toEqual("setting_name = 'Infinity'"); }); }); + describe('getSourceQuerySettings', () => { + const cacheSetting = { + setting: 'read_from_filesystem_cache_if_exists_otherwise_bypass_cache', + value: '1', + }; + const base = [{ setting: 'max_threads', value: '4' }]; + const rangeOfDays = (days: number): [Date, Date] => [ + new Date(0), + new Date(days * 24 * 60 * 60 * 1000), + ]; + + const twoDays = { filesystemCacheReadOnlyMinRange: '2 day' }; + + it('never adds the setting when the source has no threshold', () => { + expect( + getSourceQuerySettings({ querySettings: base }, rangeOfDays(30)), + ).toEqual(base); + expect(getSourceQuerySettings({}, rangeOfDays(30))).toBeUndefined(); + }); + + it('adds the setting only for ranges longer than the threshold', () => { + expect( + getSourceQuerySettings( + { ...twoDays, querySettings: base }, + rangeOfDays(2), + ), + ).toEqual(base); + expect( + getSourceQuerySettings( + { ...twoDays, querySettings: base }, + rangeOfDays(3), + ), + ).toEqual([...base, cacheSetting]); + expect(getSourceQuerySettings(twoDays, rangeOfDays(3))).toEqual([ + cacheSetting, + ]); + }); + + it('keeps a value the source sets explicitly, and skips without a range', () => { + const explicit = [{ ...cacheSetting, value: '0' }]; + expect( + getSourceQuerySettings( + { ...twoDays, querySettings: explicit }, + rangeOfDays(30), + ), + ).toEqual(explicit); + expect( + getSourceQuerySettings({ ...twoDays, querySettings: base }, undefined), + ).toEqual(base); + }); + }); describe('parseTokenizerFromTextIndex', () => { it.each([ { diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 7c93078458..2e9bcfa97a 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1430,6 +1430,39 @@ export function joinQuerySettings( return formattedPairs.join(', '); } +const FILESYSTEM_CACHE_READ_ONLY_SETTING = + 'read_from_filesystem_cache_if_exists_otherwise_bypass_cache'; + +/** + * The source's query settings for a query over `dateRange`. Pass the full + * selected range, not a chunk window, so every chunk of a windowed query gets + * the same settings. A value the source sets explicitly wins. + */ +export function getSourceQuerySettings( + source: + | Pick + | undefined, + dateRange: [Date, Date] | undefined, +): QuerySettings | undefined { + const querySettings = source?.querySettings; + const minRange = source?.filesystemCacheReadOnlyMinRange; + if ( + dateRange == null || + minRange == null || + querySettings?.some( + ({ setting }) => setting === FILESYSTEM_CACHE_READ_ONLY_SETTING, + ) || + dateRange[1].getTime() - dateRange[0].getTime() <= + convertGranularityToSeconds(minRange) * 1000 + ) { + return querySettings; + } + return [ + ...(querySettings ?? []), + { setting: FILESYSTEM_CACHE_READ_ONLY_SETTING, value: '1' }, + ]; +} + // A discriminated union type for different tokenizers above export type TextIndexTokenizer = | { type: 'splitByNonAlpha' } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index e3dd7f52a5..42a930349c 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2647,6 +2647,15 @@ export const BaseSourceSchema = z.object({ }), disabled: z.boolean().optional(), querySettings: QuerySettingsSchema.optional(), + /** + * Queries whose selected time range is longer than this read the ClickHouse + * filesystem cache without filling it, so long scans don't evict hot data. + * Unset turns it off. '' (the form's "Never" option) is treated as unset. + */ + filesystemCacheReadOnlyMinRange: z.preprocess( + v => (v === '' ? undefined : v), + SQLIntervalSchema.optional(), + ), timestampValueExpression: RequiredTimestampColumnSchema, });