diff --git a/.changeset/cascading-dashboard-filters.md b/.changeset/cascading-dashboard-filters.md
new file mode 100644
index 0000000000..5ad16612ff
--- /dev/null
+++ b/.changeset/cascading-dashboard-filters.md
@@ -0,0 +1,16 @@
+---
+'@hyperdx/app': patch
+---
+
+feat(dashboards): opt-in linked (faceted) filter values
+
+Dashboard and Kubernetes filter bars gain a "link filters" toggle (the
+bidirectional-arrow button at the end of the bar). When enabled, each filter
+dropdown only shows values that co-occur with the other current selections —
+e.g. picking a `cluster` narrows the `namespace` dropdown to namespaces in that
+cluster (the K8s bar also factors in the free-text search). A filter never
+constrains its own options, so multi-select still works. It is off by default
+because contingent value lookups can't use the cheap per-key rollups and are
+more expensive at scale; when on, all of a source's facets are computed in a
+single `groupUniqArrayIf` scan rather than one query per filter. Search-page
+filters are unaffected.
diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx
index 3095a081fd..4713dc49e4 100644
--- a/packages/app/src/DashboardFilters.tsx
+++ b/packages/app/src/DashboardFilters.tsx
@@ -1,9 +1,10 @@
-import { useMemo } from 'react';
+import { useMemo, useState } from 'react';
import { FilterState } from '@hyperdx/common-utils/dist/filters';
import { DashboardFilter } from '@hyperdx/common-utils/dist/types';
import { Group, Stack, Text, Tooltip } from '@mantine/core';
import { IconAlertTriangle, IconHelp, IconRefresh } from '@tabler/icons-react';
+import { FilterLinkToggle } from './components/FilterLinkToggle';
import { VirtualMultiSelect } from './components/VirtualMultiSelect/VirtualMultiSelect';
import { useDashboardFilterValues } from './hooks/useDashboardFilterValues';
@@ -64,10 +65,10 @@ const DashboardFilterSelect = ({
placeholder={value.length === 0 ? filter.name : undefined}
values={value}
data={valuesOrEmptyMemo}
- // Disable only while values are genuinely loading. A completed query
- // that returned no rows (or failed) must stay interactive so the user
- // can still clear/adjust the selection instead of being stuck.
- disabled={isLoading}
+ // Surface loading as a dropdown hint rather than disabling the control,
+ // so a completed/empty/failed query stays interactive and the user can
+ // still clear or adjust the selection.
+ loading={isLoading}
onChange={onChange}
data-testid={`dashboard-filter-select-${filter.name}`}
/>
@@ -89,6 +90,12 @@ const DashboardFilters = ({
filterValues,
onSetFilterValue,
}: DashboardFilterProps) => {
+ // "Link" mode (opt-in, off by default): each dropdown's values are narrowed by
+ // the others' selections. Off by default because contingent value lookups
+ // can't use the cheap per-key rollups and are more expensive at scale. When
+ // on, all of a source's facets are computed in a single groupUniqArrayIf scan.
+ const [linked, setLinked] = useState(false);
+
const {
data: filterValuesById,
erroredFilterIds,
@@ -96,6 +103,8 @@ const DashboardFilters = ({
} = useDashboardFilterValues({
filters,
dateRange,
+ // Only narrow by sibling selections when linked.
+ filterValues: linked ? filterValues : {},
});
return (
@@ -108,7 +117,7 @@ const DashboardFilters = ({
: [];
// Fall back to the hook-level fetching state only until this filter's
// query has produced an entry; once it has (even with empty values),
- // honor its own loading flag so a finished query never stays disabled.
+ // honor its own loading flag.
const isLoadingValues = queriedFilterValues
? queriedFilterValues.isLoading
: isFetching;
@@ -124,6 +133,19 @@ const DashboardFilters = ({
/>
);
})}
+ {filters.length >= 2 && (
+
+ {/* Spacer to align the toggle with the inputs (filters have a label row above). */}
+
+
+
+
+ )}
{isFetching &&
+ onChange(!linked)}
+ aria-label="Link filters"
+ aria-pressed={linked}
+ data-testid={dataTestId}
+ >
+
+
+ );
+}
diff --git a/packages/app/src/components/KubernetesFilters.tsx b/packages/app/src/components/KubernetesFilters.tsx
index 3367ad6359..7a5c003941 100644
--- a/packages/app/src/components/KubernetesFilters.tsx
+++ b/packages/app/src/components/KubernetesFilters.tsx
@@ -1,12 +1,15 @@
import React, { useEffect, useMemo, useRef, useState } from 'react';
+import { escapeRegExp } from 'lodash';
import { useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
+import { FilterState } from '@hyperdx/common-utils/dist/filters';
import {
BuilderChartConfigWithDateRange,
TMetricSource,
} from '@hyperdx/common-utils/dist/types';
import { Box, Group, Select } from '@mantine/core';
+import { FilterLinkToggle } from '@/components/FilterLinkToggle';
import SearchInputV2 from '@/components/SearchInput/SearchInputV2';
import { useGetKeyValues } from '@/hooks/useMetadata';
@@ -17,44 +20,84 @@ type KubernetesFiltersProps = {
setSearchQuery: (query: string) => void;
};
+// The bundled Kubernetes filter dropdowns, in display order.
+const K8S_FILTER_FIELDS = [
+ {
+ field: 'k8s.pod.name',
+ placeholder: 'Pod',
+ dataTestId: 'pod-filter-select',
+ },
+ {
+ field: 'k8s.deployment.name',
+ placeholder: 'Deployment',
+ dataTestId: 'deployment-filter-select',
+ },
+ {
+ field: 'k8s.node.name',
+ placeholder: 'Node',
+ dataTestId: 'node-filter-select',
+ },
+ {
+ field: 'k8s.namespace.name',
+ placeholder: 'Namespace',
+ dataTestId: 'namespace-filter-select',
+ },
+ {
+ field: 'k8s.cluster.name',
+ placeholder: 'Cluster',
+ dataTestId: 'cluster-filter-select',
+ },
+] as const;
+
type FilterSelectProps = {
- metricSource: TMetricSource;
placeholder: string;
- fieldName: string;
value: string | null;
onChange: (value: string | null) => void;
- chartConfig: BuilderChartConfigWithDateRange;
+ options: string[];
+ loading?: boolean;
dataTestId?: string;
};
+// Removes a single field's `resourceAttr.field:"..."` clause from a Lucene
+// query string, leaving every other clause (and free-text search) intact. Used
+// both to rewrite the query when a dropdown changes and to build the faceted
+// free-text `where` for the dropdown value lookup.
+export const stripFieldClause = (
+ query: string,
+ resourceAttr: string,
+ field: string,
+): string => {
+ const fullAttribute = `${resourceAttr}.${field}`;
+ const regex = new RegExp(`${escapeRegExp(fullAttribute)}:"[^"]*"`, 'g');
+ // Replace with a space and collapse runs of whitespace so removing a clause
+ // from the middle of the query doesn't leave a double space.
+ return query.replace(regex, ' ').replace(/\s+/g, ' ').trim();
+};
+
const FilterSelect: React.FC = ({
- metricSource,
placeholder,
- fieldName,
value,
onChange,
- chartConfig,
+ options,
+ loading,
dataTestId,
}) => {
- const { data, isLoading } = useGetKeyValues({
- chartConfig,
- keys: [`${metricSource.resourceAttributesExpression}['${fieldName}']`],
- disableRowLimit: true,
- limit: 1000000,
- });
-
- const options = useMemo(
- () =>
- data?.[0]?.value
- .map(value => ({ value, label: value }))
- .sort((a, b) => a.value.localeCompare(b.value)) || [], // Sort alphabetically for better search results
- [data],
- );
+ const data = useMemo(() => {
+ const opts = options
+ .map(v => ({ value: v, label: v }))
+ .sort((a, b) => a.value.localeCompare(b.value)); // Sort alphabetically for better search results
+ // Keep the current selection visible even when it isn't in the (faceted)
+ // value list, so the control still reflects what's applied.
+ if (value && !opts.some(o => o.value === value)) {
+ opts.unshift({ value, label: value });
+ }
+ return opts;
+ }, [options, value]);
return (
= ({
allowDeselect
size="xs"
maxDropdownHeight={280}
- disabled={isLoading}
variant="filled"
w={200}
limit={100} // Show up to 100 search results
@@ -84,6 +126,27 @@ export const KubernetesFilters: React.FC = ({
const [namespaceName, setNamespaceName] = useState(null);
const [clusterName, setClusterName] = useState(null);
+ // "Link" mode (opt-in, off by default): narrow each dropdown's values by the
+ // other selections + the free-text search. Off by default because contingent
+ // value lookups can't use the cheap per-key rollups and cost far more at scale.
+ const [linked, setLinked] = useState(false);
+
+ const resourceAttr = metricSource.resourceAttributesExpression;
+ const valueByField: Record = {
+ 'k8s.pod.name': podName,
+ 'k8s.deployment.name': deploymentName,
+ 'k8s.node.name': nodeName,
+ 'k8s.namespace.name': namespaceName,
+ 'k8s.cluster.name': clusterName,
+ };
+ const setterByField: Record void> = {
+ 'k8s.pod.name': setPodName,
+ 'k8s.deployment.name': setDeploymentName,
+ 'k8s.node.name': setNodeName,
+ 'k8s.namespace.name': setNamespaceName,
+ 'k8s.cluster.name': setClusterName,
+ };
+
const { control, setValue } = useForm({
defaultValues: {
searchQuery: searchQuery,
@@ -115,8 +178,9 @@ export const KubernetesFilters: React.FC = ({
resourceAttr: string = '',
attribute: string,
) => {
+ const fullAttribute = `${resourceAttr}.${attribute}`;
const match = searchQuery.match(
- new RegExp(`${resourceAttr}\\.${attribute}:"([^"]+)"`, 'i'),
+ new RegExp(`${escapeRegExp(fullAttribute)}:"([^"]+)"`, 'i'),
);
return match ? match[1] : null;
};
@@ -124,51 +188,100 @@ export const KubernetesFilters: React.FC = ({
// Initialize filter values from search query
useEffect(() => {
if (searchQuery) {
- const resourceAttr = metricSource.resourceAttributesExpression;
-
- setPodName(
- extractValueFromSearchQuery(searchQuery, resourceAttr, 'k8s.pod.name'),
- );
- setDeploymentName(
- extractValueFromSearchQuery(
- searchQuery,
- resourceAttr,
- 'k8s.deployment.name',
- ),
- );
- setNodeName(
- extractValueFromSearchQuery(searchQuery, resourceAttr, 'k8s.node.name'),
- );
- setNamespaceName(
- extractValueFromSearchQuery(
- searchQuery,
- resourceAttr,
- 'k8s.namespace.name',
- ),
- );
- setClusterName(
- extractValueFromSearchQuery(
- searchQuery,
- resourceAttr,
- 'k8s.cluster.name',
- ),
- );
+ for (const { field } of K8S_FILTER_FIELDS) {
+ setterByField[field](
+ extractValueFromSearchQuery(searchQuery, resourceAttr, field),
+ );
+ }
}
- }, [searchQuery, metricSource.resourceAttributesExpression]);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [searchQuery, resourceAttr]);
- // Create chart config for fetching key values
- const chartConfig: BuilderChartConfigWithDateRange = {
- from: {
- databaseName: metricSource.from.databaseName,
- tableName: metricSource.metricTables?.gauge || '',
- },
- where: '',
- whereLanguage: 'sql',
- select: '',
- timestampValueExpression: metricSource.timestampValueExpression || '',
- connection: metricSource.connection,
- dateRange,
- };
+ const keys = useMemo(
+ () => K8S_FILTER_FIELDS.map(({ field }) => `${resourceAttr}['${field}']`),
+ [resourceAttr],
+ );
+
+ // Faceted (linked) lookups: build a per-field constraint from the OTHER
+ // selected fields (exclude-self) so all five value lists are computed in a
+ // single `groupUniqArrayIf` scan. The free-text search is applied as a shared
+ // WHERE. When unlinked there are no constraints and every dropdown lists all
+ // values (still a single batched query).
+ //
+ // Kept as FilterState rather than SQL so getKeyValues can render the keys the
+ // same way it renders them in the SELECT.
+ const keyConditions = useMemo(() => {
+ if (!linked) return undefined;
+ return K8S_FILTER_FIELDS.map(({ field }) => {
+ const others: FilterState = {};
+ let hasSelection = false;
+ for (const { field: otherField } of K8S_FILTER_FIELDS) {
+ const value = valueByField[otherField];
+ if (otherField !== field && value) {
+ others[`${resourceAttr}['${otherField}']`] = {
+ included: new Set([value]),
+ excluded: new Set(),
+ };
+ hasSelection = true;
+ }
+ }
+ return hasSelection ? others : undefined;
+ });
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [
+ linked,
+ resourceAttr,
+ podName,
+ deploymentName,
+ nodeName,
+ namespaceName,
+ clusterName,
+ ]);
+
+ // Free-text portion of the search (everything except the structured field
+ // clauses), applied as a shared WHERE so the dropdowns honor it when linked.
+ const facetWhere = useMemo(
+ () =>
+ linked
+ ? K8S_FILTER_FIELDS.reduce(
+ (query, { field }) => stripFieldClause(query, resourceAttr, field),
+ searchQuery,
+ )
+ : '',
+ [linked, resourceAttr, searchQuery],
+ );
+
+ const chartConfig: BuilderChartConfigWithDateRange = useMemo(
+ () => ({
+ from: {
+ databaseName: metricSource.from.databaseName,
+ tableName: metricSource.metricTables?.gauge || '',
+ },
+ where: facetWhere,
+ whereLanguage: 'lucene',
+ select: '',
+ timestampValueExpression: metricSource.timestampValueExpression || '',
+ connection: metricSource.connection,
+ dateRange,
+ }),
+ [metricSource, facetWhere, dateRange],
+ );
+
+ const { data, isLoading } = useGetKeyValues({
+ chartConfig,
+ keys,
+ keyConditions,
+ disableRowLimit: true,
+ limit: 1000000,
+ });
+
+ const valuesByKey = useMemo(() => {
+ const map = new Map();
+ for (const entry of data ?? []) {
+ map.set(entry.key, entry.value as string[]);
+ }
+ return map;
+ }, [data]);
// Helper function to update search query
const updateSearchQuery = (
@@ -178,13 +291,10 @@ export const KubernetesFilters: React.FC = ({
) => {
setter(value);
- const resourceAttr = metricSource.resourceAttributesExpression;
const fullAttribute = `${resourceAttr}.${attribute}`;
// Remove existing filter for this attribute if it exists
- let newQuery = searchQuery;
- const regex = new RegExp(`${fullAttribute}:"[^"]*"`, 'g');
- newQuery = newQuery.replace(regex, '').trim();
+ let newQuery = stripFieldClause(searchQuery, resourceAttr, attribute);
// Add new filter if value is not null
if (value) {
@@ -196,62 +306,23 @@ export const KubernetesFilters: React.FC = ({
return (
- updateSearchQuery('k8s.pod.name', value, setPodName)}
- chartConfig={chartConfig}
- dataTestId="pod-filter-select"
- />
-
-
- updateSearchQuery('k8s.deployment.name', value, setDeploymentName)
- }
- chartConfig={chartConfig}
- dataTestId="deployment-filter-select"
- />
-
-
- updateSearchQuery('k8s.node.name', value, setNodeName)
- }
- chartConfig={chartConfig}
- dataTestId="node-filter-select"
- />
-
-
- updateSearchQuery('k8s.namespace.name', value, setNamespaceName)
- }
- chartConfig={chartConfig}
- dataTestId="namespace-filter-select"
- />
-
-
- updateSearchQuery('k8s.cluster.name', value, setClusterName)
- }
- chartConfig={chartConfig}
- dataTestId="cluster-filter-select"
+ {K8S_FILTER_FIELDS.map(({ field, placeholder, dataTestId }) => (
+
+ updateSearchQuery(field, value, setterByField[field])
+ }
+ options={valuesByKey.get(`${resourceAttr}['${field}']`) ?? []}
+ loading={isLoading}
+ dataTestId={dataTestId}
+ />
+ ))}
+
void;
@@ -30,6 +32,7 @@ type VirtualMultiSelectProps = {
export function VirtualMultiSelect({
data,
disabled,
+ loading,
placeholder,
values,
onChange,
@@ -217,7 +220,9 @@ export function VirtualMultiSelect({
) : (
- Nothing found...
+
+ {loading ? 'Loading…' : 'Nothing found...'}
+
)}
diff --git a/packages/app/src/components/__tests__/KubernetesFilters.test.ts b/packages/app/src/components/__tests__/KubernetesFilters.test.ts
new file mode 100644
index 0000000000..95af8c78e4
--- /dev/null
+++ b/packages/app/src/components/__tests__/KubernetesFilters.test.ts
@@ -0,0 +1,59 @@
+import { stripFieldClause } from '@/components/KubernetesFilters';
+
+describe('stripFieldClause', () => {
+ const resourceAttr = 'ResourceAttributes';
+
+ it('removes only the target field clause, keeping other filters and free text', () => {
+ const query =
+ 'ResourceAttributes.k8s.cluster.name:"prod" ResourceAttributes.k8s.namespace.name:"api" error';
+
+ // Building the faceted `where` for the namespace dropdown drops the
+ // namespace clause but keeps the cluster selection and free-text search,
+ // so the namespace options are narrowed to the selected cluster.
+ expect(stripFieldClause(query, resourceAttr, 'k8s.namespace.name')).toBe(
+ 'ResourceAttributes.k8s.cluster.name:"prod" error',
+ );
+ });
+
+ it('returns an empty string when the query only contains the target clause', () => {
+ expect(
+ stripFieldClause(
+ 'ResourceAttributes.k8s.cluster.name:"prod"',
+ resourceAttr,
+ 'k8s.cluster.name',
+ ),
+ ).toBe('');
+ });
+
+ it('leaves the query unchanged when the target field is absent', () => {
+ const query = 'ResourceAttributes.k8s.cluster.name:"prod"';
+ expect(stripFieldClause(query, resourceAttr, 'k8s.namespace.name')).toBe(
+ query,
+ );
+ });
+
+ it('does not strip a sibling field that shares a path prefix', () => {
+ // `k8s.pod.name` must not match `k8s.pod.uid`.
+ const query =
+ 'ResourceAttributes.k8s.pod.name:"a" ResourceAttributes.k8s.pod.uid:"b"';
+ expect(stripFieldClause(query, resourceAttr, 'k8s.pod.name')).toBe(
+ 'ResourceAttributes.k8s.pod.uid:"b"',
+ );
+ });
+
+ it('treats dots in the attribute as literal characters, not regex wildcards', () => {
+ // Unescaped, the dots would act as wildcards and wrongly strip this clause
+ // where the path separators are different characters. Escaping keeps them
+ // literal, so a non-dotted lookalike is left untouched.
+ const query = 'ResourceAttributesXk8sXpodXname:"a"';
+ expect(stripFieldClause(query, resourceAttr, 'k8s.pod.name')).toBe(query);
+ });
+
+ it('does not throw when the resource attribute contains regex metacharacters', () => {
+ // An unescaped `(` would produce an "unterminated group" SyntaxError.
+ expect(() =>
+ stripFieldClause('foo', 'attr(', 'k8s.pod.name'),
+ ).not.toThrow();
+ expect(stripFieldClause('foo', 'attr(', 'k8s.pod.name')).toBe('foo');
+ });
+});
diff --git a/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx
index 7fcd82b338..9de069de5d 100644
--- a/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx
+++ b/packages/app/src/hooks/__tests__/useDashboardFilterValues.test.tsx
@@ -1,7 +1,11 @@
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
import React from 'react';
-import { optimizeGetKeyValuesCalls } from '@hyperdx/common-utils/dist/core/materializedViews';
+import {
+ optimizeFacetedKeyValuesConfig,
+ optimizeGetKeyValuesCalls,
+} from '@hyperdx/common-utils/dist/core/materializedViews';
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
+import { FilterState } from '@hyperdx/common-utils/dist/filters';
import {
DashboardFilter,
MetricsDataType,
@@ -24,6 +28,10 @@ jest.mock('@hyperdx/common-utils/dist/core/materializedViews', () => ({
.mockImplementation(async ({ keys, chartConfig }) => [
{ keys, chartConfig },
]),
+ // Default: no covering MV → faceted query runs against the raw config.
+ optimizeFacetedKeyValuesConfig: jest
+ .fn()
+ .mockImplementation(async ({ chartConfig }) => chartConfig),
}));
describe('useDashboardFilterValues', () => {
@@ -1056,4 +1064,297 @@ describe('useDashboardFilterValues', () => {
}),
);
});
+
+ describe('faceted filtering (cascading filters)', () => {
+ // Returns the single arg object passed to the MOST RECENT getKeyValues call
+ // whose `keys` exactly match, or undefined if no such call was made. (Most
+ // recent matters when a rerender issues a fresh call for the same keys.)
+ const callForKeys = (keys: string[]) =>
+ (mockMetadata.getKeyValues.mock.calls as any[])
+ .filter(([arg]) => JSON.stringify(arg.keys) === JSON.stringify(keys))
+ .at(-1)?.[0];
+
+ const envAndStatus: DashboardFilter[] = [
+ {
+ id: 'filter1',
+ type: 'QUERY_EXPRESSION',
+ name: 'Environment',
+ expression: 'environment',
+ source: 'logs-source',
+ },
+ {
+ id: 'filter2',
+ type: 'QUERY_EXPRESSION',
+ name: 'Status',
+ expression: 'status',
+ source: 'logs-source',
+ },
+ ];
+
+ beforeEach(() => {
+ jest.spyOn(sourceModule, 'useSources').mockReturnValue({
+ data: mockSources,
+ isLoading: false,
+ } as any);
+ // Unconstrained groups still go through the optimizer; restore its
+ // passthrough (clearAllMocks resets call data but not implementations).
+ jest
+ .mocked(optimizeGetKeyValuesCalls)
+ .mockImplementation(async ({ keys, chartConfig }) => [
+ { keys, chartConfig },
+ ]);
+ // Default: faceted queries run against the raw config (no covering MV).
+ jest
+ .mocked(optimizeFacetedKeyValuesConfig)
+ .mockImplementation(async ({ chartConfig }) => chartConfig);
+ });
+
+ // What a filter narrowed by an `environment: production` selection should
+ // receive. The hook passes the selection down as state, not as SQL, so
+ // getKeyValues can render its keys the same way it renders the SELECT.
+ const envProductionConstraint = {
+ environment: {
+ included: new Set(['production']),
+ excluded: new Set(),
+ },
+ };
+
+ it('resolves every key in one faceted scan, constraining each by the others (exclude-self)', async () => {
+ const { result } = renderHook(
+ () =>
+ useDashboardFilterValues({
+ filters: envAndStatus,
+ dateRange: mockDateRange,
+ filterValues: {
+ environment: {
+ included: new Set(['production']),
+ excluded: new Set(),
+ },
+ },
+ }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ // A single scan for the whole source — not one query per filter.
+ expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(1);
+ // `status` is constrained by the environment selection; `environment` is
+ // NOT constrained by its own selection (exclude-self → undefined).
+ expect(callForKeys(['environment', 'status'])?.keyConditions).toEqual([
+ undefined,
+ envProductionConstraint,
+ ]);
+ });
+
+ it('runs one unconstrained query when nothing is selected', async () => {
+ const { result } = renderHook(
+ () =>
+ useDashboardFilterValues({
+ filters: envAndStatus,
+ dateRange: mockDateRange,
+ filterValues: {},
+ }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(1);
+ // No conditions → plain groupUniqArray (no keyConditions passed).
+ expect(
+ callForKeys(['environment', 'status'])?.keyConditions,
+ ).toBeUndefined();
+ });
+
+ it('still uses a single scan for many filters when one is selected', async () => {
+ const filters: DashboardFilter[] = [
+ ...envAndStatus,
+ {
+ id: 'filter3',
+ type: 'QUERY_EXPRESSION',
+ name: 'Log Level',
+ expression: 'log_level',
+ source: 'logs-source',
+ },
+ ];
+
+ const { result } = renderHook(
+ () =>
+ useDashboardFilterValues({
+ filters,
+ dateRange: mockDateRange,
+ filterValues: {
+ environment: {
+ included: new Set(['production']),
+ excluded: new Set(),
+ },
+ },
+ }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ // One faceted scan regardless of how many filters/selections.
+ expect(mockMetadata.getKeyValues).toHaveBeenCalledTimes(1);
+ expect(
+ callForKeys(['environment', 'status', 'log_level'])?.keyConditions,
+ ).toEqual([undefined, envProductionConstraint, envProductionConstraint]);
+ });
+
+ it('does not apply a selection from one source to filters on another source', async () => {
+ const filters: DashboardFilter[] = [
+ ...envAndStatus,
+ {
+ id: 'filter3',
+ type: 'QUERY_EXPRESSION',
+ name: 'Service',
+ expression: 'service.name',
+ source: 'traces-source',
+ },
+ ];
+
+ const { result } = renderHook(
+ () =>
+ useDashboardFilterValues({
+ filters,
+ dateRange: mockDateRange,
+ filterValues: {
+ environment: {
+ included: new Set(['production']),
+ excluded: new Set(),
+ },
+ },
+ }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ // The logs group is faceted (status narrowed by env)...
+ expect(callForKeys(['environment', 'status'])?.keyConditions).toEqual([
+ undefined,
+ envProductionConstraint,
+ ]);
+ // ...but the traces filter is never constrained by the logs selection.
+ expect(callForKeys(['service.name'])?.keyConditions).toBeUndefined();
+ });
+
+ it('does not apply a selection across metric types of the same source', async () => {
+ // Same source, different metric type → different physical table, so the
+ // constrained column need not exist there.
+ const filters: DashboardFilter[] = [
+ {
+ id: 'gauge-filter',
+ type: 'QUERY_EXPRESSION',
+ name: 'Gauge Metric',
+ expression: 'MetricName',
+ source: 'metric-source',
+ sourceMetricType: MetricsDataType.Gauge,
+ },
+ {
+ id: 'sum-filter',
+ type: 'QUERY_EXPRESSION',
+ name: 'Sum Metric',
+ expression: 'SumMetricName',
+ source: 'metric-source',
+ sourceMetricType: MetricsDataType.Sum,
+ },
+ ];
+
+ const { result } = renderHook(
+ () =>
+ useDashboardFilterValues({
+ filters,
+ dateRange: mockDateRange,
+ filterValues: {
+ MetricName: {
+ included: new Set(['CPU_Usage']),
+ excluded: new Set(),
+ },
+ },
+ }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ expect(callForKeys(['MetricName'])?.keyConditions).toBeUndefined();
+ expect(callForKeys(['SumMetricName'])?.keyConditions).toBeUndefined();
+ });
+
+ it('refetches with updated conditions when a selection changes', async () => {
+ const { result, rerender } = renderHook(
+ ({ filterValues }) =>
+ useDashboardFilterValues({
+ filters: envAndStatus,
+ dateRange: mockDateRange,
+ filterValues,
+ }),
+ {
+ wrapper,
+ initialProps: { filterValues: {} as FilterState },
+ },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+ expect(
+ callForKeys(['environment', 'status'])?.keyConditions,
+ ).toBeUndefined();
+
+ rerender({
+ filterValues: {
+ environment: {
+ included: new Set(['production']),
+ excluded: new Set(),
+ },
+ },
+ });
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ expect(callForKeys(['environment', 'status'])?.keyConditions).toEqual([
+ undefined,
+ envProductionConstraint,
+ ]);
+ });
+
+ it('runs the faceted scan against a covering materialized view when one is found', async () => {
+ // Simulate a covering MV: the resolver points the faceted query at the
+ // rollup table.
+ jest
+ .mocked(optimizeFacetedKeyValuesConfig)
+ .mockImplementation(async ({ chartConfig }) => ({
+ ...chartConfig,
+ from: { databaseName: 'telemetry', tableName: 'logs_rollup_1m' },
+ }));
+
+ const { result } = renderHook(
+ () =>
+ useDashboardFilterValues({
+ filters: envAndStatus,
+ dateRange: mockDateRange,
+ filterValues: {
+ environment: {
+ included: new Set(['production']),
+ excluded: new Set(),
+ },
+ },
+ }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isFetching).toBe(false));
+
+ const call = callForKeys(['environment', 'status']);
+ // The single faceted scan targets the rollup, still carrying the per-key
+ // conditions.
+ expect(call?.chartConfig?.from).toEqual({
+ databaseName: 'telemetry',
+ tableName: 'logs_rollup_1m',
+ });
+ expect(call?.keyConditions).toEqual([undefined, envProductionConstraint]);
+ });
+ });
});
diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx
index 0d952644a5..1b25ff518c 100644
--- a/packages/app/src/hooks/useDashboardFilterValues.tsx
+++ b/packages/app/src/hooks/useDashboardFilterValues.tsx
@@ -2,8 +2,13 @@ import { useMemo } from 'react';
import { pick } from 'lodash';
import {
GetKeyValueCall,
+ optimizeFacetedKeyValuesConfig,
optimizeGetKeyValuesCalls,
} from '@hyperdx/common-utils/dist/core/materializedViews';
+import {
+ FilterState,
+ serializeFilterState,
+} from '@hyperdx/common-utils/dist/filters';
import {
BuilderChartConfigWithDateRange,
DashboardFilter,
@@ -37,49 +42,109 @@ const filterToKey = (filter: DashboardFilter): string =>
whereLanguage: filter.whereLanguage ?? 'sql',
} satisfies FilterSourceKey);
-const filterFromKey = (key: string): FilterSourceKey =>
- JSON.parse(key) as FilterSourceKey;
-
type EnrichedCall = GetKeyValueCall & {
/** filterIds[i] = array of filter IDs whose values come from keys[i] */
filterIds: string[][];
+ /** Per-key constraint for faceted lookups, aligned with `keys`. */
+ keyConditions?: (FilterState | undefined)[];
};
function useOptimizedKeyValuesCalls({
filters,
dateRange,
+ filterValues,
}: {
filters: DashboardFilter[];
dateRange: [Date, Date];
+ filterValues: FilterState;
}) {
const clickhouseClient = useClickhouseClient();
const metadata = useMetadataWithSettings();
const { data: sources, isLoading: isLoadingSources } = useSources();
- // Group filters by (source, metricType, where, whereLanguage) so that we can test each group for MV compatibility separately.
+ // Faceted filtering: each filter's selectable values are narrowed by the
+ // CURRENT selections of its sibling filters. For every filter, collect the
+ // selections of the OTHER filters that target the same source + metric type
+ // (so the constrained columns exist in the queried table), EXCLUDING the
+ // filter's own expression (otherwise a multi-select would collapse to only
+ // its already-selected values). Passing this down as a per-key constraint
+ // lets all of a source's filters resolve in one `groupUniqArrayIf` scan
+ // instead of one query per filter.
+ //
+ // The constraint stays a FilterState rather than SQL: getKeyValues renders
+ // it against the same key expressions it puts in the SELECT, which raw SQL
+ // built here could not line up with on JSON columns.
+ const constraintByFilterId = useMemo(() => {
+ const byId = new Map();
+ for (const filter of filters) {
+ const prunedState: FilterState = {};
+ let hasSelection = false;
+ for (const sibling of filters) {
+ if (
+ sibling.source !== filter.source ||
+ sibling.sourceMetricType !== filter.sourceMetricType ||
+ // Exclude-self: FilterState is keyed by expression, so a sibling that
+ // shares this filter's expression carries this filter's own selection.
+ sibling.expression === filter.expression
+ ) {
+ continue;
+ }
+ const selection = filterValues[sibling.expression];
+ if (
+ selection &&
+ (selection.included.size > 0 ||
+ selection.excluded.size > 0 ||
+ selection.range != null)
+ ) {
+ prunedState[sibling.expression] = selection;
+ hasSelection = true;
+ }
+ }
+ byId.set(filter.id, hasSelection ? prunedState : undefined);
+ }
+ return byId;
+ }, [filters, filterValues]);
+
+ // Group filters by (source, metricType, where, whereLanguage). Every filter in
+ // a group is resolved together: an unconstrained group goes through the MV
+ // optimizer (one batched, rollup-eligible query); a constrained group runs a
+ // single faceted `groupUniqArrayIf` scan carrying a per-key condition.
const filtersByGroupKey = useMemo(() => {
- const filtersByGroupKey = new Map();
+ const byGroupKey = new Map();
for (const filter of filters) {
const key = filterToKey(filter);
- if (!filtersByGroupKey.has(key)) {
- filtersByGroupKey.set(key, [filter]);
+ const existing = byGroupKey.get(key);
+ if (existing) {
+ existing.push(filter);
} else {
- filtersByGroupKey.get(key)!.push(filter);
+ byGroupKey.set(key, [filter]);
}
}
- return filtersByGroupKey;
+ return byGroupKey;
}, [filters]);
const results: UseQueryResult[] = useQueries({
- queries: Array.from(filtersByGroupKey.entries())
- .filter(([key]) =>
- sources?.some(s => s.id === filterFromKey(key).sourceId),
+ queries: Array.from(filtersByGroupKey.values())
+ .filter(filtersInGroup =>
+ sources?.some(s => s.id === filtersInGroup[0].source),
)
- .map(([key, filtersInGroup]) => {
- const { sourceId, metricType, where, whereLanguage } =
- filterFromKey(key);
+ .map(filtersInGroup => {
+ const representative = filtersInGroup[0];
+ const sourceId = representative.source;
+ const metricType = representative.sourceMetricType;
+ const where = representative.where ?? '';
+ const whereLanguage = representative.whereLanguage ?? 'sql';
const source = sources!.find(s => s.id === sourceId)!;
const keyExpressions = filtersInGroup.map(f => f.expression);
+ const keyConditions = filtersInGroup.map(f =>
+ constraintByFilterId.get(f.id),
+ );
+ const isFaceted = keyConditions.some(c => c != null);
+ // Sets don't survive react-query's JSON.stringify key hashing, so the
+ // key carries a serialized projection instead of the raw state.
+ const keyConditionsSignature = keyConditions.map(
+ c => c && serializeFilterState(c),
+ );
const tableName = getMetricTableName(source, metricType) ?? '';
const chartConfig: BuilderChartConfigWithDateRange = {
@@ -107,6 +172,13 @@ function useOptimizedKeyValuesCalls({
select: '',
};
+ const filterIdsForKeys = (keys: string[]) =>
+ keys.map(expression =>
+ filtersInGroup
+ .filter(f => f.expression === expression)
+ .map(f => f.id),
+ );
+
return {
queryKey: [
'dashboard-filters-key-value-calls',
@@ -116,10 +188,35 @@ function useOptimizedKeyValuesCalls({
keyExpressions,
where,
whereLanguage,
+ keyConditionsSignature,
],
enabled: !isLoadingSources,
staleTime: 1000 * 60 * 5, // Cache every 5 min
- queryFn: async ({ signal }) => {
+ queryFn: async ({ signal }): Promise => {
+ // Constrained: resolve every key in one faceted scan
+ // (groupUniqArrayIf). A per-key condition can't be split across
+ // single-key MVs, but it can run against one MV whose dimensions
+ // cover every filter column (else the raw table).
+ if (isFaceted) {
+ const facetedConfig = await optimizeFacetedKeyValuesConfig({
+ chartConfig,
+ keys: keyExpressions,
+ keyConditions,
+ source,
+ clickhouseClient,
+ metadata,
+ signal,
+ });
+ return [
+ {
+ chartConfig: facetedConfig,
+ keys: keyExpressions,
+ keyConditions,
+ filterIds: filterIdsForKeys(keyExpressions),
+ },
+ ];
+ }
+ // Unconstrained: let the MV optimizer batch / route to rollups.
const calls = await optimizeGetKeyValuesCalls({
chartConfig,
source,
@@ -128,14 +225,9 @@ function useOptimizedKeyValuesCalls({
keys: keyExpressions,
signal,
});
- // Enrich each call with the filter IDs that correspond to each key expression
return calls.map(call => ({
...call,
- filterIds: call.keys.map(expression =>
- filtersInGroup
- .filter(f => f.expression === expression)
- .map(f => f.id),
- ),
+ filterIds: filterIdsForKeys(call.keys),
}));
},
};
@@ -152,9 +244,11 @@ function useOptimizedKeyValuesCalls({
export function useDashboardFilterValues({
filters,
dateRange,
+ filterValues = {},
}: {
filters: DashboardFilter[];
dateRange: [Date, Date];
+ filterValues?: FilterState;
}) {
const metadata = useMetadataWithSettings();
const {
@@ -164,6 +258,7 @@ export function useDashboardFilterValues({
} = useOptimizedKeyValuesCalls({
filters,
dateRange,
+ filterValues,
});
const { data: sources, isLoading: isSourcesLoading } = useSources();
@@ -173,7 +268,7 @@ export function useDashboardFilterValues({
type TQueryData = { key: string; value: string[] }[];
const results: UseQueryResult[] = useQueries({
- queries: calls.map(({ chartConfig, keys }) => {
+ queries: calls.map(({ chartConfig, keys, keyConditions }) => {
// Construct a query key prefix which will allow us to use placeholder data from the previous query for the same keys
const queryKeyPrefix = [
'dashboard-filter-key-values',
@@ -184,7 +279,13 @@ export function useDashboardFilterValues({
const source = sourcesLookup.get(chartConfig.source);
return {
- queryKey: [...queryKeyPrefix, chartConfig],
+ queryKey: [
+ ...queryKeyPrefix,
+ chartConfig,
+ // Serialized: react-query hashes keys with JSON.stringify, which
+ // would flatten every distinct Set selection to `{}`.
+ keyConditions?.map(c => c && serializeFilterState(c)),
+ ],
placeholderData: () => {
// Use placeholder data from the most recently cached query with the same key prefix
const cached = queryClient
@@ -206,6 +307,7 @@ export function useDashboardFilterValues({
metadata.getKeyValues({
chartConfig,
keys,
+ keyConditions,
limit: 10000,
disableRowLimit: true,
signal,
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index 8522202b69..fca1559d5d 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -11,6 +11,10 @@ import {
TableConnection,
TableMetadata,
} from '@hyperdx/common-utils/dist/core/metadata';
+import {
+ FilterState,
+ serializeFilterState,
+} from '@hyperdx/common-utils/dist/filters';
import {
BuilderChartConfigWithDateRange,
isLogSource,
@@ -337,6 +341,7 @@ export function useMultipleGetKeyValues(
{
chartConfigs,
keys,
+ keyConditions,
limit,
disableRowLimit,
mode = 'exact',
@@ -346,6 +351,8 @@ export function useMultipleGetKeyValues(
| BuilderChartConfigWithDateRange
| BuilderChartConfigWithDateRange[];
keys: string[];
+ /** Per-key constraints for faceted ('exact' mode) value lookups. */
+ keyConditions?: (FilterState | undefined)[];
limit?: number;
disableRowLimit?: boolean;
mode?: 'all' | 'exact';
@@ -371,6 +378,9 @@ export function useMultipleGetKeyValues(
metadataMVsOverride,
...chartConfigsArr.map(cc => ({ ...cc })),
...keys,
+ // Serialized: react-query hashes keys with JSON.stringify, which would
+ // flatten every distinct Set selection to `{}`.
+ keyConditions?.map(c => c && serializeFilterState(c)),
disableRowLimit,
maxKeys,
],
@@ -417,6 +427,7 @@ export function useMultipleGetKeyValues(
return metadata.getKeyValuesWithMVs({
chartConfig,
keys: keys.slice(0, maxKeys),
+ keyConditions: keyConditions?.slice(0, maxKeys),
limit,
disableRowLimit,
source,
@@ -477,6 +488,7 @@ export function useGetKeyValues(
{
chartConfig,
keys,
+ keyConditions,
limit,
disableRowLimit,
mode,
@@ -484,6 +496,8 @@ export function useGetKeyValues(
}: {
chartConfig?: BuilderChartConfigWithDateRange;
keys: string[];
+ /** Per-key constraints for faceted value lookups (groupUniqArrayIf). */
+ keyConditions?: (FilterState | undefined)[];
limit?: number;
disableRowLimit?: boolean;
mode?: 'all' | 'exact';
@@ -495,6 +509,7 @@ export function useGetKeyValues(
{
chartConfigs: chartConfig ? [chartConfig] : [],
keys,
+ keyConditions,
limit,
disableRowLimit,
mode,
diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts
index 702aa2e570..595c8f624e 100644
--- a/packages/common-utils/src/__tests__/filters.test.ts
+++ b/packages/common-utils/src/__tests__/filters.test.ts
@@ -1,7 +1,10 @@
import {
+ FilterState,
+ filterStateToPredicate,
filtersToQuery,
isRenderablePinnedFilter,
parseQuery,
+ serializeFilterState,
validateDashboardFilterQueries,
validateSavedFilterValues,
validateSavedQuery,
@@ -357,6 +360,119 @@ describe('filters', () => {
});
});
+ describe('filterStateToPredicate', () => {
+ const identity = (k: string) => k;
+
+ it('returns undefined when nothing is selected', () => {
+ expect(filterStateToPredicate({}, identity)).toBeUndefined();
+ expect(
+ filterStateToPredicate(
+ { colA: { included: new Set(), excluded: new Set() } },
+ identity,
+ ),
+ ).toBeUndefined();
+ });
+
+ it('wraps a single condition in parentheses', () => {
+ expect(
+ filterStateToPredicate(
+ { colA: { included: new Set(['x']), excluded: new Set() } },
+ identity,
+ ),
+ ).toBe("(colA IN ('x'))");
+ });
+
+ it('AND-joins conditions across keys and across include/exclude', () => {
+ const predicate = filterStateToPredicate(
+ {
+ colA: { included: new Set(['x']), excluded: new Set(['y']) },
+ colB: { included: new Set(['z']), excluded: new Set() },
+ },
+ identity,
+ );
+ expect(predicate).toBe(
+ "(colA IN ('x')) AND (colA NOT IN ('y')) AND (colB IN ('z'))",
+ );
+ });
+
+ it('addresses each key through renderKey', () => {
+ // The whole point of the callback: a JSON column is aggregated as a
+ // typed subcolumn, so the predicate has to name it the same way.
+ expect(
+ filterStateToPredicate(
+ {
+ "Attributes['cluster']": {
+ included: new Set(['prod']),
+ excluded: new Set(),
+ },
+ },
+ () => '`Attributes`.`cluster`.:String',
+ ),
+ ).toBe("(`Attributes`.`cluster`.:String IN ('prod'))");
+ });
+ });
+
+ describe('serializeFilterState', () => {
+ it('distinguishes different selections', () => {
+ const a: FilterState = {
+ colA: { included: new Set(['x']), excluded: new Set() },
+ };
+ const b: FilterState = {
+ colA: { included: new Set(['y']), excluded: new Set() },
+ };
+ expect(serializeFilterState(a)).not.toBe(serializeFilterState(b));
+ // JSON.stringify alone would flatten both Sets to {} and collide, which
+ // is exactly what this helper exists to prevent in react-query keys.
+ expect(JSON.stringify(a)).toBe(JSON.stringify(b));
+ });
+
+ it('distinguishes an included value from an excluded one', () => {
+ expect(
+ serializeFilterState({
+ colA: { included: new Set(['x']), excluded: new Set() },
+ }),
+ ).not.toBe(
+ serializeFilterState({
+ colA: { included: new Set(), excluded: new Set(['x']) },
+ }),
+ );
+ });
+
+ it('is stable across key and member insertion order', () => {
+ expect(
+ serializeFilterState({
+ colA: { included: new Set(['x', 'y']), excluded: new Set() },
+ colB: { included: new Set(['z']), excluded: new Set() },
+ }),
+ ).toBe(
+ serializeFilterState({
+ colB: { included: new Set(['z']), excluded: new Set() },
+ colA: { included: new Set(['y', 'x']), excluded: new Set() },
+ }),
+ );
+ });
+
+ it('includes the range bound', () => {
+ expect(
+ serializeFilterState({
+ colA: {
+ included: new Set(),
+ excluded: new Set(),
+ range: { min: 1, max: 2 },
+ },
+ }),
+ ).not.toBe(
+ serializeFilterState({
+ colA: {
+ included: new Set(),
+ excluded: new Set(),
+ range: { min: 1, max: 3 },
+ },
+ }),
+ );
+ });
+ });
+
describe('validateSavedFilterValues', () => {
it('returns no issues for an empty array', () => {
expect(validateSavedFilterValues([])).toEqual([]);
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 94d4d38772..f238321e89 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1,3 +1,4 @@
+import { ColumnMeta } from '@/clickhouse';
import { ClickhouseClient } from '@/clickhouse/node';
import {
GET_ALL_KEY_VALUES_CHUNK_SIZE,
@@ -619,6 +620,84 @@ describe('Metadata', () => {
);
});
+ it('uses groupUniqArrayIf for keys that have a condition (faceted lookup)', async () => {
+ const renderChartConfigSpy = jest.spyOn(
+ renderChartConfigModule,
+ 'renderChartConfig',
+ );
+
+ await metadata.getKeyValues({
+ chartConfig: mockChartConfig,
+ keys: ['colA', 'colB'],
+ keyConditions: [
+ undefined,
+ { colA: { included: new Set(['x']), excluded: new Set() } },
+ ],
+ limit: 10,
+ disableRowLimit: true,
+ source,
+ });
+
+ const actualConfig = renderChartConfigSpy.mock.calls.at(-1)![0];
+ if (!isBuilderChartConfig(actualConfig))
+ throw new Error('Expected builder config');
+ // No condition → plain groupUniqArray; condition → groupUniqArrayIf so
+ // both facets resolve in a single scan.
+ expect(actualConfig.select).toContain(
+ 'groupUniqArray(10)(colA) AS param0',
+ );
+ expect(actualConfig.select).toContain(
+ "groupUniqArrayIf(10)(colB, (colA IN ('x'))) AS param1",
+ );
+ });
+
+ it('renders faceted conditions the same way as the keys they constrain', async () => {
+ // A JSON column renders to a typed subcolumn rather than the bracket
+ // form the caller wrote. If the predicate kept the raw expression it
+ // would address a different (or non-existent) column than the aggregate
+ // wrapping it, so both halves must come out rendered.
+ const jsonMetadata = new Metadata(mockClickhouseClient, mockCache);
+ jest
+ .spyOn(jsonMetadata, 'getColumn')
+ .mockImplementation(async ({ column }) =>
+ column === 'Attributes'
+ ? ({ name: 'Attributes', type: 'JSON' } as ColumnMeta)
+ : undefined,
+ );
+
+ const renderChartConfigSpy = jest.spyOn(
+ renderChartConfigModule,
+ 'renderChartConfig',
+ );
+
+ await jsonMetadata.getKeyValues({
+ chartConfig: mockChartConfig,
+ keys: ["Attributes['cluster']", "Attributes['namespace']"],
+ keyConditions: [
+ undefined,
+ {
+ "Attributes['cluster']": {
+ included: new Set(['prod']),
+ excluded: new Set(),
+ },
+ },
+ ],
+ limit: 10,
+ disableRowLimit: true,
+ source,
+ });
+
+ const actualConfig = renderChartConfigSpy.mock.calls.at(-1)![0];
+ if (!isBuilderChartConfig(actualConfig))
+ throw new Error('Expected builder config');
+ expect(actualConfig.select).toContain(
+ 'groupUniqArrayIf(10)(Attributes.`namespace`.:String, ' +
+ "(Attributes.`cluster`.:String IN ('prod'))) AS param1",
+ );
+ // The raw bracket form must not survive into either half.
+ expect(actualConfig.select).not.toContain("Attributes['");
+ });
+
it('should apply row limit by default when disableRowLimit is not specified', async () => {
await metadata.getKeyValues({
chartConfig: mockChartConfig,
diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts
index 95bfc32c56..f25ad35367 100644
--- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts
+++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts
@@ -2,6 +2,7 @@ import { ColumnMeta } from '@/clickhouse';
import { ClickhouseClient } from '@/clickhouse/node';
import {
isUnsupportedCountFunction,
+ optimizeFacetedKeyValuesConfig,
optimizeGetKeyValuesCalls,
tryConvertConfigToMaterializedViewSelect,
tryOptimizeConfigWithMaterializedView,
@@ -41,6 +42,12 @@ describe('materializedViews', () => {
};
return columns[column];
}),
+ // These fixtures use plain identifiers, which render to themselves.
+ renderKeyExpressions: jest
+ .fn()
+ .mockImplementation(({ keys }: { keys: string[] }) =>
+ Promise.resolve(new Map(keys.map(k => [k, k]))),
+ ),
} as unknown as Metadata;
const MV_CONFIG_METRIC_ROLLUP_1M: MaterializedViewConfiguration = {
@@ -2544,4 +2551,122 @@ describe('materializedViews', () => {
expect(result[0].keys).toEqual(['environment', 'service']);
});
});
+
+ describe('optimizeFacetedKeyValuesConfig', () => {
+ const mockClickHouseClient = {
+ testChartConfigValidity: jest.fn(),
+ } as unknown as jest.Mocked;
+
+ const facetedChartConfig: ChartConfigWithOptDateRange = {
+ from: { databaseName: 'default', tableName: 'otel_spans' },
+ select: '',
+ where: '',
+ connection: 'test-connection',
+ };
+
+ const facetedSource = {
+ kind: SourceKind.Log,
+ from: { databaseName: 'default', tableName: 'otel_spans' },
+ materializedViews: [MV_CONFIG_METRIC_ROLLUP_1M],
+ } as TLogSource;
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
+ isValid: true,
+ rowEstimate: 1000,
+ error: undefined,
+ });
+ });
+
+ it('routes to a covering MV and EXPLAINs the faceted select', async () => {
+ const result = await optimizeFacetedKeyValuesConfig({
+ chartConfig: facetedChartConfig,
+ keys: ['ServiceName', 'StatusCode'],
+ keyConditions: [
+ undefined,
+ { ServiceName: { included: new Set(['x']), excluded: new Set() } },
+ ],
+ source: facetedSource,
+ clickhouseClient: mockClickHouseClient,
+ metadata,
+ });
+
+ // The faceted query runs against the rollup, using its timestamp column.
+ expect(result.from).toEqual({
+ databaseName: 'default',
+ tableName: 'metrics_rollup_1m',
+ });
+ expect(result.timestampValueExpression).toBe('Timestamp');
+ // Validation used groupUniqArrayIf only for the constrained key.
+ expect(mockClickHouseClient.testChartConfigValidity).toHaveBeenCalledWith(
+ expect.objectContaining({
+ config: expect.objectContaining({
+ from: { databaseName: 'default', tableName: 'metrics_rollup_1m' },
+ select:
+ "groupUniqArray(1)(ServiceName) AS param0, groupUniqArrayIf(1)(StatusCode, (ServiceName IN ('x'))) AS param1",
+ }),
+ }),
+ );
+ });
+
+ it('falls back to the raw table when a key is not an MV dimension', async () => {
+ const result = await optimizeFacetedKeyValuesConfig({
+ chartConfig: facetedChartConfig,
+ keys: ['ServiceName', 'NotADimension'],
+ keyConditions: [
+ undefined,
+ { ServiceName: { included: new Set(['x']), excluded: new Set() } },
+ ],
+ source: facetedSource,
+ clickhouseClient: mockClickHouseClient,
+ metadata,
+ });
+
+ expect(result).toBe(facetedChartConfig);
+ expect(
+ mockClickHouseClient.testChartConfigValidity,
+ ).not.toHaveBeenCalled();
+ });
+
+ it('falls back to the raw table when the MV EXPLAIN is invalid', async () => {
+ mockClickHouseClient.testChartConfigValidity.mockResolvedValue({
+ isValid: false,
+ error: '',
+ });
+
+ const result = await optimizeFacetedKeyValuesConfig({
+ chartConfig: facetedChartConfig,
+ keys: ['ServiceName', 'StatusCode'],
+ keyConditions: [
+ undefined,
+ { ServiceName: { included: new Set(['x']), excluded: new Set() } },
+ ],
+ source: facetedSource,
+ clickhouseClient: mockClickHouseClient,
+ metadata,
+ });
+
+ expect(result).toBe(facetedChartConfig);
+ });
+
+ it('returns the raw table for a source without materialized views', async () => {
+ const result = await optimizeFacetedKeyValuesConfig({
+ chartConfig: facetedChartConfig,
+ keys: ['ServiceName'],
+ keyConditions: [undefined],
+ source: {
+ kind: SourceKind.Log,
+ from: { databaseName: 'default', tableName: 'otel_spans' },
+ } as TLogSource,
+ clickhouseClient: mockClickHouseClient,
+ metadata,
+ });
+
+ expect(result).toBe(facetedChartConfig);
+ expect(
+ mockClickHouseClient.testChartConfigValidity,
+ ).not.toHaveBeenCalled();
+ });
+ });
});
diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts
index 810dd79063..e7ba7d132d 100644
--- a/packages/common-utils/src/core/materializedViews.ts
+++ b/packages/common-utils/src/core/materializedViews.ts
@@ -1,6 +1,7 @@
import { differenceInSeconds } from 'date-fns';
import { BaseClickhouseClient, ChSql, chSql } from '@/clickhouse';
+import { FilterState, filterStateToPredicate } from '@/filters';
import {
BuilderChartConfigWithOptDateRange,
CteChartConfig,
@@ -899,3 +900,115 @@ export async function optimizeGetKeyValuesCalls<
return calls;
}
+
+/**
+ * Resolve which table a *faceted* key-values lookup should scan. A faceted
+ * lookup applies a different predicate per key (`groupUniqArrayIf`), so it can't
+ * be split across single-key rollups like `optimizeGetKeyValuesCalls` — but it
+ * CAN run against one materialized view whose dimension columns cover every
+ * filter column. Returns an MV-pointed chartConfig when a covering MV validates
+ * via EXPLAIN (cheapest row estimate wins), otherwise the original (raw) config.
+ */
+export async function optimizeFacetedKeyValuesConfig<
+ C extends BuilderChartConfigWithOptDateRange,
+>({
+ chartConfig,
+ keys,
+ keyConditions,
+ source,
+ clickhouseClient,
+ metadata,
+ signal,
+}: {
+ chartConfig: C;
+ keys: string[];
+ /** Per-key constraint aligned with `keys` (undefined = unconstrained). */
+ keyConditions: (FilterState | undefined)[];
+ source: TSource | undefined;
+ clickhouseClient: BaseClickhouseClient;
+ metadata: Metadata;
+ signal?: AbortSignal;
+}): Promise {
+ const mvs =
+ source && (isTraceSource(source) || isLogSource(source))
+ ? (source.materializedViews ?? [])
+ : [];
+ if (mvs.length === 0) return chartConfig;
+
+ // A covering MV must expose every filter column as a dimension. The per-key
+ // conditions only reference other keys, so requiring all `keys` covers them
+ // too; anything else (e.g. the static `where`) is caught by the EXPLAIN below.
+ //
+ // Matched against the RAW keys on purpose: `dimensionColumns` is configured
+ // by hand in the source form using the same expressions the filters use, not
+ // the rendered physical form. Only the generated SQL below is rendered.
+ const coveringMvs = mvs.filter(mv => {
+ const intervalsInDateRange = chartConfig.dateRange
+ ? countIntervalsInDateRange(chartConfig.dateRange, mv.minGranularity)
+ : Infinity;
+ if (
+ !mvConfigSupportsDateRange(mv, chartConfig) ||
+ intervalsInDateRange < 3
+ ) {
+ return false;
+ }
+ const dimensionColumns = splitAndTrimWithBracket(mv.dimensionColumns);
+ return keys.every(k => dimensionColumns.includes(k));
+ });
+ if (coveringMvs.length === 0) return chartConfig;
+
+ // Build + EXPLAIN-validate a faceted config against each covering MV. The
+ // EXPLAIN only needs the same columns, so the limit is irrelevant here.
+ const explainResults = await Promise.all(
+ coveringMvs.map(async mv => {
+ // Render against the MV, since that is the table getKeyValues will
+ // render against once this config is returned. An EXPLAIN built from raw
+ // expressions would validate different SQL than the one we end up running.
+ const renderedByKey = await metadata.renderKeyExpressions({
+ databaseName: mv.databaseName,
+ tableName: mv.tableName,
+ connectionId: chartConfig.connection,
+ keys,
+ });
+ const renderKey = (key: string) => renderedByKey.get(key) ?? key;
+ const config = {
+ ...structuredClone(chartConfig),
+ timestampValueExpression: mv.timestampColumn,
+ from: { databaseName: mv.databaseName, tableName: mv.tableName },
+ select: keys
+ .map((key, i) => {
+ const k = renderKey(key);
+ const state = keyConditions.at(i);
+ const condition = state && filterStateToPredicate(state, renderKey);
+ return condition
+ ? `groupUniqArrayIf(1)(${k}, ${condition}) AS param${i}`
+ : `groupUniqArray(1)(${k}) AS param${i}`;
+ })
+ .join(', '),
+ };
+ const { isValid, rowEstimate = Number.POSITIVE_INFINITY } =
+ await clickhouseClient.testChartConfigValidity({
+ config,
+ metadata,
+ opts: { abort_signal: signal },
+ querySettings: source?.querySettings,
+ });
+ return { mv, isValid, rowEstimate };
+ }),
+ );
+
+ const best = explainResults
+ .filter(r => r.isValid)
+ .sort((a, b) => a.rowEstimate - b.rowEstimate)[0];
+ if (!best) return chartConfig;
+
+ const optimizedConfig: C = {
+ ...structuredClone(chartConfig),
+ timestampValueExpression: best.mv.timestampColumn,
+ from: {
+ databaseName: best.mv.databaseName,
+ tableName: best.mv.tableName,
+ },
+ };
+ return optimizedConfig;
+}
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index eddd1675b2..c443ce58f7 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -14,6 +14,11 @@ import {
tableExpr,
} from '@/clickhouse';
import { renderChartConfig, timeFilterExpr } from '@/core/renderChartConfig';
+import {
+ FilterState,
+ filterStateToPredicate,
+ serializeFilterState,
+} from '@/filters';
import {
buildTextIndexInfoLookup,
skipIndexMatches,
@@ -33,6 +38,7 @@ import {
supportsMergeTreeTextIndex,
} from './clickhouseVersion';
import {
+ optimizeFacetedKeyValuesConfig,
optimizeGetKeyValuesCalls,
renderStartOfBucketExpr,
} from './materializedViews';
@@ -334,6 +340,43 @@ export class Metadata {
return keyExpression;
}
+ /**
+ * Batch-render key expressions to their physical SQL form, keyed by the raw
+ * expression the caller passed in.
+ *
+ * Public so that callers building SQL which must line up with what
+ * `getKeyValues` emits — e.g. the materialized-view EXPLAIN probe in
+ * `optimizeFacetedKeyValuesConfig` — can render the same way rather than
+ * interpolating the raw expressions and diverging on JSON columns.
+ */
+ async renderKeyExpressions({
+ databaseName,
+ tableName,
+ connectionId,
+ keys,
+ }: {
+ databaseName: string;
+ tableName: string;
+ connectionId: string;
+ keys: string[];
+ }): Promise> {
+ const rendered = new Map();
+ await Promise.all(
+ keys.map(async key => {
+ rendered.set(
+ key,
+ await this.renderMetadataKeyExpression({
+ databaseName,
+ tableName,
+ connectionId,
+ keyExpression: key,
+ }),
+ );
+ }),
+ );
+ return rendered;
+ }
+
private async queryTableMetadata({
database,
table,
@@ -2388,6 +2431,7 @@ export class Metadata {
async getKeyValues({
chartConfig,
keys,
+ keyConditions,
limit = 20,
disableRowLimit = false,
signal,
@@ -2395,6 +2439,19 @@ export class Metadata {
}: {
chartConfig: BuilderChartConfigWithDateRange;
keys: string[];
+ /**
+ * Optional per-key constraint, aligned with `keys`: the selections that
+ * should narrow this key's values. When provided for a key, its values are
+ * gathered with `groupUniqArrayIf(key, )` so several "faceted"
+ * value lists (each constrained differently) resolve in a single table
+ * scan. Only applied when `disableRowLimit` is true (the path used by
+ * filter dropdowns).
+ *
+ * Keys inside the state are RAW expressions; they are rendered here the
+ * same way as `keys`, so the predicate addresses the same physical column
+ * as the aggregate wrapping it.
+ */
+ keyConditions?: (FilterState | undefined)[];
limit?: number;
disableRowLimit?: boolean;
signal?: AbortSignal;
@@ -2412,6 +2469,8 @@ export class Metadata {
'filters',
]),
keys,
+ // Serialize rather than hashing the raw state: the selections are Sets.
+ keyConditions: keyConditions?.map(s => s && serializeFilterState(s)),
disableRowLimit,
};
return this.cache.getOrFetch(
@@ -2419,24 +2478,39 @@ export class Metadata {
async () => {
if (keys.length === 0) return [];
- const renderedKeys = await Promise.all(
- keys.map(key =>
- this.renderMetadataKeyExpression({
- databaseName: chartConfig.from.databaseName,
- tableName: chartConfig.from.tableName,
- connectionId: chartConfig.connection,
- keyExpression: key,
- }),
- ),
- );
+ // A constraint only ever references sibling keys from this same call,
+ // so the map always covers it; `?? key` is a defensive fallback rather
+ // than an expected path.
+ const renderedByKey = await this.renderKeyExpressions({
+ databaseName: chartConfig.from.databaseName,
+ tableName: chartConfig.from.tableName,
+ connectionId: chartConfig.connection,
+ keys,
+ });
+ const renderKey = (key: string) => renderedByKey.get(key) ?? key;
+ const renderedKeys = keys.map(renderKey);
// When disableRowLimit is true, query directly without CTE
// Otherwise, use CTE with row limits for sampling
const sqlConfig = disableRowLimit
? {
...chartConfig,
- select: renderedKeys
- .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
+ select: keys
+ .map((key, i) => {
+ const k = renderKey(key);
+ const state = keyConditions?.at(i);
+ // Render the constraint against the same expressions used in
+ // the SELECT, or the predicate would address a different
+ // physical column than the aggregate wrapping it.
+ const condition =
+ state && filterStateToPredicate(state, renderKey);
+ // `groupUniqArrayIf` lets each key be constrained by its own
+ // predicate, so multiple faceted value lists are computed in a
+ // single scan instead of one query per key.
+ return condition
+ ? `groupUniqArrayIf(${limit})(${k}, ${condition}) AS param${i}`
+ : `groupUniqArray(${limit})(${k}) AS param${i}`;
+ })
.join(', '),
}
: await (async () => {
@@ -2519,6 +2593,7 @@ export class Metadata {
async getKeyValuesWithMVs({
chartConfig,
keys,
+ keyConditions,
source,
limit = 20,
disableRowLimit,
@@ -2526,6 +2601,8 @@ export class Metadata {
}: {
chartConfig: BuilderChartConfigWithDateRange;
keys: string[];
+ /** Per-key constraints for faceted value lookups; see getKeyValues. */
+ keyConditions?: (FilterState | undefined)[];
source: TSource | undefined;
limit?: number;
disableRowLimit?: boolean;
@@ -2541,6 +2618,8 @@ export class Metadata {
'filters',
]),
keys,
+ // Serialize rather than hashing the raw state: the selections are Sets.
+ keyConditions: keyConditions?.map(s => s && serializeFilterState(s)),
disableRowLimit,
};
return this.cache.getOrFetch(
@@ -2548,6 +2627,31 @@ export class Metadata {
async () => {
if (keys.length === 0) return [];
+ // Faceted lookups apply a different predicate per key, so they can't be
+ // split across single-key materialized views — but they can run as one
+ // scan against a materialized view whose dimensions cover every filter
+ // column (else fall back to the raw table).
+ if (keyConditions && keyConditions.some(c => c != null)) {
+ const facetedConfig = await optimizeFacetedKeyValuesConfig({
+ chartConfig,
+ keys,
+ keyConditions,
+ source,
+ clickhouseClient: this.clickhouseClient,
+ metadata: this,
+ signal,
+ });
+ return this.getKeyValues({
+ chartConfig: facetedConfig,
+ keys,
+ keyConditions,
+ limit,
+ disableRowLimit,
+ signal,
+ source,
+ });
+ }
+
const defaultKeyValueCall = { chartConfig, keys };
const canHaveMVs =
source &&
diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts
index e15e138b01..3bc425baea 100644
--- a/packages/common-utils/src/filters.ts
+++ b/packages/common-utils/src/filters.ts
@@ -100,6 +100,61 @@ export const filtersToQuery = (
});
};
+/**
+ * Render a FilterState as a single AND-joined SQL predicate, remapping every
+ * key through `renderKey` first.
+ *
+ * Callers that also emit the same keys elsewhere in the query (e.g. inside a
+ * SELECT aggregate) must render both halves the same way, or the predicate
+ * silently addresses a different expression than the one being aggregated.
+ * `stringifyKeys` is deliberately false: a rendered JSON path already carries
+ * the `.:String` type suffix, so it needs no `toString()` wrapper.
+ *
+ * Returns undefined when nothing is selected, so callers can branch on
+ * "constrained vs unconstrained" without inspecting the string.
+ */
+export const filterStateToPredicate = (
+ state: FilterState,
+ renderKey: (rawKey: string) => string,
+): string | undefined => {
+ const rendered: FilterState = {};
+ for (const [rawKey, selection] of Object.entries(state)) {
+ rendered[renderKey(rawKey)] = selection;
+ }
+ const conditions = filtersToQuery(rendered).flatMap(f =>
+ // filtersToQuery only emits `sql` filters (which carry `condition`); the
+ // `in` guard narrows away the `sql_ast` member of the Filter union.
+ 'condition' in f ? [f.condition] : [],
+ );
+ return conditions.length
+ ? conditions.map(c => `(${c})`).join(' AND ')
+ : undefined;
+};
+
+/**
+ * Stable, JSON-safe projection of a FilterState, for use in react-query keys.
+ *
+ * A raw FilterState cannot be used as a query key: its selections are `Set`s,
+ * and TanStack Query hashes keys with JSON.stringify, which serializes any Set
+ * to `{}` — so every distinct selection would collide on one cache entry.
+ * Keys and members are sorted so that insertion order alone never produces a
+ * spurious cache miss.
+ */
+export const serializeFilterState = (state: FilterState): string => {
+ const sortMembers = (values: Set) =>
+ Array.from(values).map(String).sort();
+ return JSON.stringify(
+ Object.entries(state)
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+ .map(([key, { included, excluded, range }]) => [
+ key,
+ sortMembers(included),
+ sortMembers(excluded),
+ range ?? null,
+ ]),
+ );
+};
+
// Helper function to parse a string value as boolean if possible, or otherwise
// return as string with surrounding quotes removed and SQL-escaped quotes unescaped.
const getBooleanOrUnquotedString = (value: string): string | boolean => {