feat: skip filesystem cache writes for long time ranges

.changeset/smart-filesystem-cache.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/common-utils': minor
+'@hyperdx/api': minor
+'@hyperdx/app': minor
+---
+
+feat: New per-source setting "Skip filesystem cache writes". Charts and searches over a time range longer than the chosen threshold set `read_from_filesystem_cache_if_exists_otherwise_bypass_cache = 1`, so long scans don't evict recently cached data. Off by default.

packages/api/src/models/source.ts

@@ -111,6 +111,11 @@ const sourceBaseSchema = new Schema<MongooseSourceBase>(
         message: '{PATH} exceeds the limit of 10',
       },
     },
+    // Mirrors the '' -> undefined preprocess in BaseSourceSchema.
+    filesystemCacheReadOnlyMinRange: {
+      type: String,
+      set: (v: string | null | undefined) => (v === '' ? undefined : v),
+    },
   },
   {
     discriminatorKey: 'kind',

packages/app/src/components/Sources/SourceForm/SourceForm.tsx

@@ -40,6 +40,7 @@ import {
   AutocompleteControlled,
   InputControlled,
 } from '@/components/InputControlled';
+import SelectControlled from '@/components/SelectControlled';
 import {
   IS_METRICS_ENABLED,
   IS_PROMQL_ENABLED,
@@ -62,6 +63,7 @@ import {
 
 import {
   DEFAULT_DATABASE,
+  FILESYSTEM_CACHE_READ_ONLY_MIN_RANGE_OPTIONS,
   PROMETHEUS_PLACEHOLDER,
   SOURCE_KIND_LABELS,
 } from './constants';
@@ -746,6 +748,20 @@ export function TableSourceForm({
             Add Setting
           </Button>
         </FormRow>
+        <FormRow
+          label="Skip filesystem cache writes"
+          helpText="Queries over a selected time range longer than this set read_from_filesystem_cache_if_exists_otherwise_bypass_cache = 1, so long scans read the filesystem cache but don't fill it and evict recent data. A value set in Query Settings takes precedence."
+        >
+          <SelectControlled
+            control={control}
+            name="filesystemCacheReadOnlyMinRange"
+            data={FILESYSTEM_CACHE_READ_ONLY_MIN_RANGE_OPTIONS}
+            allowDeselect={false}
+            // Unset matches no option ('' is "Never"'s value), so the
+            // placeholder covers it.
+            placeholder={FILESYSTEM_CACHE_READ_ONLY_MIN_RANGE_OPTIONS[0].label}
+          />
+        </FormRow>
       </Stack>
       <TableModelForm
         control={control}

packages/app/src/components/Sources/SourceForm/constants.ts

@@ -50,6 +50,15 @@ export const MIN_AUTO_GRANULARITY_OPTIONS = [
   ...MV_GRANULARITY_OPTIONS.filter(option => option.value !== '1 second'),
 ];
 
+export const FILESYSTEM_CACHE_READ_ONLY_MIN_RANGE_OPTIONS = [
+  { value: '', label: 'Never' },
+  { value: '6 hour', label: 'Longer than 6 hours' },
+  { value: '1 day', label: 'Longer than 1 day' },
+  { value: '2 day', label: 'Longer than 2 days' },
+  { value: '7 day', label: 'Longer than 7 days' },
+  { value: '30 day', label: 'Longer than 30 days' },
+];
+
 export const USE_TEXT_INDEX_OPTIONS = [
   {
     value: UseTextIndex.Auto,

packages/app/src/hooks/__tests__/useChartConfig.test.tsx

@@ -25,6 +25,7 @@ import {
   useQueriedChartConfig,
 } from '@/hooks/useChartConfig';
 import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanation';
+import * as sourceModule from '@/source';
 
 // Mock DEFAULT_TIME_WINDOWS_SECONDS to remove the 15m window
 jest.mock('@/utils/searchWindows', () => {
@@ -1672,6 +1673,45 @@ describe('useChartConfig', () => {
       expect(result.current.isPending).toBe(false);
     });
 
+    it('applies the filesystem cache setting to every chunk based on the full range', async () => {
+      // 3 days, split into windows that are each shorter than the 2-day threshold.
+      const config = createMockChartConfig({
+        dateRange: [
+          new Date('2025-10-01 00:00:00Z'),
+          new Date('2025-10-04 00:00:00Z'),
+        ],
+        granularity: '1 hour',
+      });
+      mockClickhouseClient.queryChartConfig.mockResolvedValue(
+        createMockQueryResponse([]),
+      );
+      const useSourceSpy = jest
+        .spyOn(sourceModule, 'useSource')
+        .mockReturnValue({
+          data: { filesystemCacheReadOnlyMinRange: '2 day' },
+          isLoading: false,
+        } as any);
+
+      const { result } = renderHook(
+        () => useQueriedChartConfig(config, { enableQueryChunking: true }),
+        { wrapper },
+      );
+      await waitFor(() => expect(result.current.isFetching).toBe(false));
+      useSourceSpy.mockRestore();
+
+      const calls = mockClickhouseClient.queryChartConfig.mock.calls;
+      expect(calls.length).toBeGreaterThan(1);
+      for (const [{ querySettings }] of calls) {
+        expect(querySettings).toEqual([
+          {
+            setting:
+              'read_from_filesystem_cache_if_exists_otherwise_bypass_cache',
+            value: '1',
+          },
+        ]);
+      }
+    });
+
     it('pins the series-limit ranking to the newest window on each chunk', async () => {
       const config = createMockChartConfig({
         dateRange: [

packages/app/src/hooks/useChartConfig.tsx

@@ -19,6 +19,7 @@ import {
 import {
   convertDateRangeToGranularityString,
   convertGranularityToSeconds,
+  getSourceQuerySettings,
   hasPositiveSeriesLimit,
 } from '@hyperdx/common-utils/dist/core/utils';
 import {
@@ -449,7 +450,8 @@ export function useQueriedChartConfig(
         enableParallelQueries: options?.enableParallelQueries,
         metadata,
         querySettings: mergeQuerySettings(
-          source?.querySettings,
+          // The full range, before fetchDataInChunks splits it into windows.
+          getSourceQuerySettings(source, config.dateRange),
           options?.additionalQuerySettings,
         ),
       });

packages/app/src/hooks/useOffsetPaginatedQuery.tsx

@@ -15,6 +15,7 @@ import {
 import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
 import {
+  getSourceQuerySettings,
   isFirstOrderByAscending,
   isTimestampExpressionInFirstOrderBy,
   MAX_TABLE_ROWS,
@@ -257,7 +258,8 @@ const queryFn: QueryFunction<TQueryFnData, TQueryKey, TPageParam> = async ({
   const query = await renderChartConfig(
     windowedConfig,
     metadata,
-    source?.querySettings,
+    // The full range, not the page's time window.
+    getSourceQuerySettings(source, config.dateRange),
   );
 
   // Create abort signal from timeout if provided

packages/common-utils/src/__tests__/utils.test.ts

@@ -15,6 +15,7 @@ import {
   getDistributedTableArgs,
   getFirstOrderingItem,
   getPercentileAggFnLevel,
+  getSourceQuerySettings,
   hasNonEmptyOrderBy,
   hasPositiveSeriesLimit,
   isFirstOrderByAscending,
@@ -3099,6 +3100,57 @@ describe('utils', () => {
       ).toEqual("setting_name = 'Infinity'");
     });
   });
+  describe('getSourceQuerySettings', () => {
+    const cacheSetting = {
+      setting: 'read_from_filesystem_cache_if_exists_otherwise_bypass_cache',
+      value: '1',
+    };
+    const base = [{ setting: 'max_threads', value: '4' }];
+    const rangeOfDays = (days: number): [Date, Date] => [
+      new Date(0),
+      new Date(days * 24 * 60 * 60 * 1000),
+    ];
+
+    const twoDays = { filesystemCacheReadOnlyMinRange: '2 day' };
+
+    it('never adds the setting when the source has no threshold', () => {
+      expect(
+        getSourceQuerySettings({ querySettings: base }, rangeOfDays(30)),
+      ).toEqual(base);
+      expect(getSourceQuerySettings({}, rangeOfDays(30))).toBeUndefined();
+    });
+
+    it('adds the setting only for ranges longer than the threshold', () => {
+      expect(
+        getSourceQuerySettings(
+          { ...twoDays, querySettings: base },
+          rangeOfDays(2),
+        ),
+      ).toEqual(base);
+      expect(
+        getSourceQuerySettings(
+          { ...twoDays, querySettings: base },
+          rangeOfDays(3),
+        ),
+      ).toEqual([...base, cacheSetting]);
+      expect(getSourceQuerySettings(twoDays, rangeOfDays(3))).toEqual([
+        cacheSetting,
+      ]);
+    });
+
+    it('keeps a value the source sets explicitly, and skips without a range', () => {
+      const explicit = [{ ...cacheSetting, value: '0' }];
+      expect(
+        getSourceQuerySettings(
+          { ...twoDays, querySettings: explicit },
+          rangeOfDays(30),
+        ),
+      ).toEqual(explicit);
+      expect(
+        getSourceQuerySettings({ ...twoDays, querySettings: base }, undefined),
+      ).toEqual(base);
+    });
+  });
   describe('parseTokenizerFromTextIndex', () => {
     it.each([
       {

packages/common-utils/src/core/utils.ts

@@ -1430,6 +1430,39 @@ export function joinQuerySettings(
   return formattedPairs.join(', ');
 }
 
+const FILESYSTEM_CACHE_READ_ONLY_SETTING =
+  'read_from_filesystem_cache_if_exists_otherwise_bypass_cache';
+
+/**
+ * The source's query settings for a query over `dateRange`. Pass the full
+ * selected range, not a chunk window, so every chunk of a windowed query gets
+ * the same settings. A value the source sets explicitly wins.
+ */
+export function getSourceQuerySettings(
+  source:
+    | Pick<TSource, 'querySettings' | 'filesystemCacheReadOnlyMinRange'>
+    | undefined,
+  dateRange: [Date, Date] | undefined,
+): QuerySettings | undefined {
+  const querySettings = source?.querySettings;
+  const minRange = source?.filesystemCacheReadOnlyMinRange;
+  if (
+    dateRange == null ||
+    minRange == null ||
+    querySettings?.some(
+      ({ setting }) => setting === FILESYSTEM_CACHE_READ_ONLY_SETTING,
+    ) ||
+    dateRange[1].getTime() - dateRange[0].getTime() <=
+      convertGranularityToSeconds(minRange) * 1000
+  ) {
+    return querySettings;
+  }
+  return [
+    ...(querySettings ?? []),
+    { setting: FILESYSTEM_CACHE_READ_ONLY_SETTING, value: '1' },
+  ];
+}
+
 // A discriminated union type for different tokenizers above
 export type TextIndexTokenizer =
   | { type: 'splitByNonAlpha' }

packages/common-utils/src/types.ts

@@ -2647,6 +2647,15 @@ export const BaseSourceSchema = z.object({
   }),
   disabled: z.boolean().optional(),
   querySettings: QuerySettingsSchema.optional(),
+  /**
+   * Queries whose selected time range is longer than this read the ClickHouse
+   * filesystem cache without filling it, so long scans don't evict hot data.
+   * Unset turns it off. '' (the form's "Never" option) is treated as unset.
+   */
+  filesystemCacheReadOnlyMinRange: z.preprocess(
+    v => (v === '' ? undefined : v),
+    SQLIntervalSchema.optional(),
+  ),
   timestampValueExpression: RequiredTimestampColumnSchema,
 });