diff --git a/.changeset/service-map-group-by.md b/.changeset/service-map-group-by.md new file mode 100644 index 0000000000..31b522011e --- /dev/null +++ b/.changeset/service-map-group-by.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/app': minor +--- + +feat: split the service map by up to 3 extra group-by SQL expressions (for +example `ResourceAttributes['k8s.pod.name']`), so a node is a service plus +those values and a slow or failing pod is no longer averaged into its service. diff --git a/packages/app/src/DBServiceMapPage.tsx b/packages/app/src/DBServiceMapPage.tsx index 56d954d62a..8945769d16 100644 --- a/packages/app/src/DBServiceMapPage.tsx +++ b/packages/app/src/DBServiceMapPage.tsx @@ -34,6 +34,10 @@ import { PageLayout } from '@/components/PageLayout'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; +import { parseServiceMapGroupBy } from '@/components/ServiceMap/dimensions'; +import ServiceMapGroupByInput, { + ServiceMapSearchFormValues, +} from '@/components/ServiceMap/ServiceMapGroupByInput'; import { IS_LOCAL_MODE } from '@/config'; import { useGetKeyValues } from '@/hooks/useMetadata'; import { useResolvedSourceParam } from '@/hooks/useResolvedSourceParam'; @@ -86,6 +90,8 @@ const searchQueryStateMap = { where: parseAsStringEncoded, whereLanguage: parseAsStringEnum<'sql' | 'lucene'>(['sql', 'lucene']), services: parseAsArrayOf(parseAsString), + // Comma-separated SQL expressions the nodes are split by, after ServiceName. + groupBy: parseAsStringEncoded, }; function DBServiceMapPage() { @@ -121,14 +127,22 @@ function DBServiceMapPage() { // useResolvedSourceParam will show a notification to the user. const source = paramSource ?? defaultSource; - const { control, handleSubmit, setValue } = useForm({ - values: { - source: source?.id, - where: searchedConfig.where ?? '', - whereLanguage: - searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', - }, - }); + const groupBy = useMemo( + () => parseServiceMapGroupBy(searchedConfig.groupBy), + [searchedConfig.groupBy], + ); + const { control, handleSubmit, setValue } = + useForm({ + values: { + source: source?.id, + where: searchedConfig.where ?? '', + whereLanguage: + searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', + // Seeded from the parsed list so the editor shows exactly what the map + // is grouped by, even for a hand-edited URL with too many expressions. + groupBy: groupBy.join(', '), + }, + }); const watchedSource = useWatch({ control, name: 'source' }); const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = @@ -207,10 +221,17 @@ function DBServiceMapPage() { [setSearchedConfig], ); + // The time range is applied together with the form values, so an invalid + // group-by leaves the URL untouched instead of moving the time window. const onSubmit = useCallback(() => { - onSearch(displayedTimeInputValue); - handleSubmit(({ where, whereLanguage }) => { - setSearchedConfig(prev => ({ ...prev, where, whereLanguage })); + handleSubmit(({ where, whereLanguage, groupBy }) => { + onSearch(displayedTimeInputValue); + setSearchedConfig(prev => ({ + ...prev, + where, + whereLanguage, + groupBy: groupBy || null, + })); })(); }, [handleSubmit, setSearchedConfig, displayedTimeInputValue, onSearch]); @@ -410,6 +431,13 @@ function DBServiceMapPage() { sqlPlaceholder="SQL WHERE to filter spans (ex. Duration > 1000000)" minWidth="min(500px, 100%)" /> + 0 ? selectedServices : undefined } + groupBy={groupBy} onFocusService={onFocusService} /> diff --git a/packages/app/src/components/ServiceMap/ServiceMap.tsx b/packages/app/src/components/ServiceMap/ServiceMap.tsx index e8a5f3a6ce..c00ba1c416 100644 --- a/packages/app/src/components/ServiceMap/ServiceMap.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMap.tsx @@ -4,6 +4,7 @@ import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { TTraceSource } from '@hyperdx/common-utils/dist/types'; import { Box, + Button, Center, Code, Loader, @@ -28,6 +29,7 @@ import { } from '@xyflow/react'; import { SQLPreview } from '@/components/ChartSQLPreview'; +import EmptyState from '@/components/EmptyState'; import useServiceMap, { ServiceAggregation } from '@/hooks/useServiceMap'; import { useResolvedColorScheme } from '@/useUserPreferences'; @@ -38,8 +40,10 @@ import { ServiceMapMetricMax, } from './ServiceMapMetricContext'; import ServiceMapNode, { ServiceMapNodeData } from './ServiceMapNode'; +import { useServiceMapSizeGate } from './useServiceMapSizeGate'; import { getServiceMetricValue, + MAX_SERVICE_MAP_NODES, SERVICE_MAP_METRIC_LABEL, SERVICE_MAP_METRICS, ServiceMapMetric, @@ -96,6 +100,8 @@ interface ServiceMapPresentationProps { dateRange: [Date, Date]; source: TTraceSource; isSingleTrace?: boolean; + // Extra group-by SQL expressions the nodes are split by (beyond ServiceName). + groupBy: string[]; // The single service the map is currently scoped to (via a node's focus // action), or undefined when showing all/multiple services. focusedService?: string; @@ -109,6 +115,7 @@ function ServiceMapPresentation({ dateRange, source, isSingleTrace, + groupBy, focusedService, onFocusService, }: ServiceMapPresentationProps) { @@ -117,6 +124,7 @@ function ServiceMapPresentation({ const { fitView } = useReactFlow(); const colorScheme = useResolvedColorScheme(); const [metric, setMetric] = useState('errorRate'); + const { tooLarge, showAnyway } = useServiceMapSizeGate(services); // Fit the data to the viewport whenever input service information changes useEffect(() => { @@ -166,15 +174,24 @@ function ServiceMapPresentation({ }, [metric, hasLatencyData]); useEffect(() => { + // Skip the layout entirely for an oversized map; dagre on thousands of + // nodes is what locks the tab, not the query. + if (tooLarge) { + setNodes([]); + setEdges([]); + return; + } + const nodes: Node[] = Array.from(services?.values() ?? []).map((service, index) => ({ - id: service.serviceName, + id: service.key, data: { ...service, dateRange, source, maxThroughput: metricMax.throughput, isSingleTrace, + groupBy, focusedService, onFocusService, }, @@ -186,16 +203,16 @@ function ServiceMapPresentation({ services?.values() ?? [], ) .filter(service => service.incomingRequestsByClient.size > 0) - .flatMap(({ serviceName, incomingRequestsByClient: requestsByClient }) => - Array.from(requestsByClient.entries()).map( + .flatMap(({ key, serviceName, dimensions, incomingRequestsByClient }) => + Array.from(incomingRequestsByClient.entries()).map( ([ - clientServiceName, + clientKey, { totalRequests, errorPercentage, p50, p95, p99, hasLatency }, ]) => { return { - id: `${serviceName}-${clientServiceName}`, - source: clientServiceName, - target: serviceName, + id: `${clientKey}->${key}`, + source: clientKey, + target: key, animated: true, type: 'request', data: { @@ -208,6 +225,8 @@ function ServiceMapPresentation({ source, dateRange, serviceName, + dimensions, + groupBy, isSingleTrace, }, }; @@ -221,10 +240,12 @@ function ServiceMapPresentation({ setEdges(edges); }, [ services, + tooLarge, dateRange, source, metricMax.throughput, isSingleTrace, + groupBy, focusedService, onFocusService, ]); @@ -248,6 +269,29 @@ function ServiceMapPresentation({ ); } + if (services && tooLarge) { + return ( +
+ + + +
+ ); + } + if (error) { return ( @@ -336,11 +380,17 @@ interface ServiceMapProps { where?: string; whereLanguage?: 'sql' | 'lucene'; serviceNames?: string[]; + // Extra group-by SQL expressions; ServiceName is always the first key. + groupBy?: string[]; // Called when a node is clicked, e.g. to drive the service filter to focus // on that service and its immediate dependencies. onFocusService?: (serviceName: string) => void; } +// A stable default so the layout effect does not re-run on every render when +// the caller passes no group-by. +const NO_GROUP_BY: string[] = []; + export default function ServiceMap({ traceId, traceTableSource, @@ -350,6 +400,7 @@ export default function ServiceMap({ where, whereLanguage, serviceNames, + groupBy = NO_GROUP_BY, onFocusService, }: ServiceMapProps) { const { @@ -364,6 +415,7 @@ export default function ServiceMap({ where, whereLanguage, serviceNames, + groupBy, }); useEffect(() => { @@ -390,6 +442,7 @@ export default function ServiceMap({ dateRange={dateRange} source={traceTableSource} isSingleTrace={isSingleTrace} + groupBy={groupBy} focusedService={focusedService} onFocusService={onFocusService} /> diff --git a/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx b/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx index 9e08473531..b55e34b7a5 100644 --- a/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMapEdge.tsx @@ -20,7 +20,10 @@ export type ServiceMapEdgeData = { hasLatency: boolean; dateRange: [Date, Date]; source: TTraceSource; + // The server (target) node's label parts. serviceName: string; + dimensions: string[]; + groupBy: string[]; isSingleTrace?: boolean; }; @@ -42,6 +45,8 @@ export default function ServiceMapEdge( hasLatency, dateRange, serviceName, + dimensions, + groupBy, source, isSingleTrace, } = props.data; @@ -70,6 +75,8 @@ export default function ServiceMapEdge( source={source} dateRange={dateRange} serviceName={serviceName} + dimensions={dimensions} + groupBy={groupBy} isSingleTrace={isSingleTrace} /> diff --git a/packages/app/src/components/ServiceMap/ServiceMapGroupByInput.tsx b/packages/app/src/components/ServiceMap/ServiceMapGroupByInput.tsx new file mode 100644 index 0000000000..368027c200 --- /dev/null +++ b/packages/app/src/components/ServiceMap/ServiceMapGroupByInput.tsx @@ -0,0 +1,72 @@ +import { Control, useFormState } from 'react-hook-form'; +import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata'; +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { Box, Text } from '@mantine/core'; + +import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; +import { MAX_SERVICE_MAP_GROUP_BY } from '@/hooks/useServiceMap'; + +export type ServiceMapSearchFormValues = { + source?: string; + where: string; + whereLanguage: 'sql' | 'lucene'; + // Comma-separated extra group-by SQL expressions. + groupBy: string; +}; + +function validateGroupBy(value: string): true | string { + return ( + splitAndTrimWithBracket(value).length <= MAX_SERVICE_MAP_GROUP_BY || + `Up to ${MAX_SERVICE_MAP_GROUP_BY} group by expressions` + ); +} + +/** The "Group by" SQL editor, with the expression-count validation inline. */ +export default function ServiceMapGroupByInput({ + control, + tableConnection, + sourceId, + dateRange, + onSubmit, +}: { + control: Control; + tableConnection: TableConnection; + sourceId: string; + dateRange: [Date, Date]; + onSubmit: () => void; +}) { + const { errors } = useFormState({ control, name: 'groupBy' }); + + return ( + + + {errors.groupBy?.message ? ( + + {errors.groupBy.message} + + ) : null} + + ); +} diff --git a/packages/app/src/components/ServiceMap/ServiceMapNode.tsx b/packages/app/src/components/ServiceMap/ServiceMapNode.tsx index c0a2219340..7936959756 100644 --- a/packages/app/src/components/ServiceMap/ServiceMapNode.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMapNode.tsx @@ -5,6 +5,7 @@ import { Handle, Node, NodeProps, NodeToolbar, Position } from '@xyflow/react'; import { ServiceAggregation } from '@/hooks/useServiceMap'; +import { formatServiceMapDimensions } from './dimensions'; import { ServiceMapMetricContext } from './ServiceMapMetricContext'; import ServiceMapTooltip from './ServiceMapTooltip'; import { @@ -23,6 +24,8 @@ export type ServiceMapNodeData = ServiceAggregation & { // scale node size. maxThroughput: number; isSingleTrace?: boolean; + // The extra group-by SQL expressions behind `dimensions`. + groupBy: string[]; // Name of the service the map is currently focused on (if any), so a node can // tell whether it is the focused one and show the matching CTA. focusedService?: string; @@ -36,6 +39,8 @@ export default function ServiceMapNode( const { data } = props; const { serviceName, + dimensions, + groupBy, incomingRequests: { totalRequests: totalIncomingRequestCount, errorPercentage, @@ -83,6 +88,7 @@ export default function ServiceMapNode( totalIncomingRequestCount + outgoingRequests, maxThroughput, ); + const dimensionsLabel = formatServiceMapDimensions(dimensions); return ( <> @@ -95,6 +101,8 @@ export default function ServiceMapNode( source={source} dateRange={dateRange} serviceName={serviceName} + dimensions={dimensions} + groupBy={groupBy} isSingleTrace={isSingleTrace} isFocused={focusedService === serviceName} onFocus={ @@ -133,6 +141,15 @@ export default function ServiceMapNode( {serviceName} + {dimensionsLabel ? ( + + {dimensionsLabel} + + ) : null} ); diff --git a/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx b/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx index 9996df3d71..f2bc0fa039 100644 --- a/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx +++ b/packages/app/src/components/ServiceMap/ServiceMapTooltip.tsx @@ -13,6 +13,10 @@ import { import { formatDurationMs } from '@/utils'; +import { + buildServiceMapDimensionsWhere, + formatServiceMapDimensions, +} from './dimensions'; import { buildServiceMapErrorWhere, ERROR_RATE_HIGH, @@ -31,6 +35,8 @@ export default function ServiceMapTooltip({ source, dateRange, serviceName, + dimensions, + groupBy, isSingleTrace, isFocused, onFocus, @@ -44,6 +50,10 @@ export default function ServiceMapTooltip({ source: TTraceSource; dateRange: [Date, Date]; serviceName: string; + // Values of the extra group-by expressions for this node, in groupBy order. + dimensions: string[]; + // The extra group-by SQL expressions, so search links can pin each value. + groupBy: string[]; isSingleTrace?: boolean; // Whether the map is currently scoped to this service (the focus toggle is on) // — flips the focus CTA to a "clear" action. @@ -63,30 +73,35 @@ export default function ServiceMapTooltip({ ? 'var(--color-text-danger)' : 'var(--color-chart-warning)'; + const dimensionsLabel = formatServiceMapDimensions(dimensions); + const dimensionsWhere = buildServiceMapDimensionsWhere(groupBy, dimensions); + const handleRequestsClick = useCallback(() => { navigateToTraceSearch({ dateRange, source, - where: SqlString.format("? = ? AND ? IN ('Server', 'Consumer')", [ - SqlString.raw(source.serviceNameExpression ?? 'ServiceName'), - serviceName, - SqlString.raw(source.spanKindExpression ?? 'SpanKind'), - ]), + where: + SqlString.format("? = ? AND ? IN ('Server', 'Consumer')", [ + SqlString.raw(source.serviceNameExpression ?? 'ServiceName'), + serviceName, + SqlString.raw(source.spanKindExpression ?? 'SpanKind'), + ]) + dimensionsWhere, }); - }, [dateRange, source, serviceName]); + }, [dateRange, source, serviceName, dimensionsWhere]); const handleErrorsClick = useCallback(() => { navigateToTraceSearch({ dateRange, source, - where: buildServiceMapErrorWhere({ - serviceNameExpression: source.serviceNameExpression ?? 'ServiceName', - serviceName, - spanKindExpression: source.spanKindExpression ?? 'SpanKind', - statusCodeExpression: source.statusCodeExpression ?? 'StatusCode', - }), + where: + buildServiceMapErrorWhere({ + serviceNameExpression: source.serviceNameExpression ?? 'ServiceName', + serviceName, + spanKindExpression: source.spanKindExpression ?? 'SpanKind', + statusCodeExpression: source.statusCodeExpression ?? 'StatusCode', + }) + dimensionsWhere, }); - }, [dateRange, source, serviceName]); + }, [dateRange, source, serviceName, dimensionsWhere]); const showMetrics = totalRequests > 0 && (requestsPerSecond != null || latencyMs != null); @@ -104,6 +119,18 @@ export default function ServiceMapTooltip({ > {serviceName} + {dimensionsLabel ? ( + + {dimensionsLabel} + + ) : null}