diff --git a/.changeset/khaki-adults-study.md b/.changeset/khaki-adults-study.md new file mode 100644 index 0000000000..2bbacadd9d --- /dev/null +++ b/.changeset/khaki-adults-study.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": minor +--- + +feat: Add metrics attribute explorer in chart builder diff --git a/packages/app/src/components/DBEditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm.tsx index 9742f156c3..76fbaa5209 100644 --- a/packages/app/src/components/DBEditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm.tsx @@ -80,7 +80,11 @@ import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor'; import { TimePicker } from '@/components/TimePicker'; import { IS_LOCAL_MODE } from '@/config'; import { GranularityPickerControlled } from '@/GranularityPicker'; -import { useFetchMetricResourceAttrs } from '@/hooks/useFetchMetricResourceAttrs'; +import { useFetchMetricMetadata } from '@/hooks/useFetchMetricMetadata'; +import { + parseAttributeKeysFromSuggestions, + useFetchMetricResourceAttrs, +} from '@/hooks/useFetchMetricResourceAttrs'; import SearchInputV2 from '@/SearchInputV2'; import { getFirstTimestampValueExpression, useSource } from '@/source'; import { @@ -112,6 +116,7 @@ import { InputControlled, TextInputControlled, } from './InputControlled'; +import { MetricAttributeHelperPanel } from './MetricAttributeHelperPanel'; import { MetricNameSelect } from './MetricNameSelect'; import SaveToDashboardModal from './SaveToDashboardModal'; import SourceSchemaPreview from './SourceSchemaPreview'; @@ -248,15 +253,54 @@ function ChartSeriesEditorComponent({ : _tableName; const metricName = useWatch({ control, name: `${namePrefix}metricName` }); - const { data: attributeKeys } = useFetchMetricResourceAttrs({ + const aggCondition = useWatch({ + control, + name: `${namePrefix}aggCondition`, + }); + const groupBy = useWatch({ control, name: 'groupBy' }); + + const { data: attributeSuggestions, isLoading: isLoadingAttributes } = + useFetchMetricResourceAttrs({ + databaseName, + metricType, + metricName, + tableSource, + isSql: aggConditionLanguage === 'sql', + }); + + const attributeKeys = useMemo( + () => parseAttributeKeysFromSuggestions(attributeSuggestions ?? []), + [attributeSuggestions], + ); + + const { data: metricMetadata } = useFetchMetricMetadata({ databaseName, - tableName: tableName || '', metricType, metricName, tableSource, - isSql: aggConditionLanguage === 'sql', }); + const handleAddToWhere = useCallback( + (clause: string) => { + const currentValue = aggCondition || ''; + + const newValue = currentValue ? `${currentValue} AND ${clause}` : clause; + setValue(`${namePrefix}aggCondition`, newValue); + onSubmit(); + }, + [aggCondition, namePrefix, setValue, onSubmit], + ); + + const handleAddToGroupBy = useCallback( + (clause: string) => { + const currentValue = groupBy || ''; + const newValue = currentValue ? `${currentValue}, ${clause}` : clause; + setValue('groupBy', newValue); + onSubmit(); + }, + [groupBy, setValue, onSubmit], + ); + const showWhere = aggFn !== 'none'; const tableConnection = useMemo( @@ -414,7 +458,7 @@ function ChartSeriesEditorComponent({ onLanguageChange={lang => setValue(`${namePrefix}aggConditionLanguage`, lang) } - additionalSuggestions={attributeKeys} + additionalSuggestions={attributeSuggestions} language="sql" onSubmit={onSubmit} /> @@ -429,7 +473,7 @@ function ChartSeriesEditorComponent({ language="lucene" placeholder="Search your events w/ Lucene ex. column:foo" onSubmit={onSubmit} - additionalSuggestions={attributeKeys} + additionalSuggestions={attributeSuggestions} /> )} @@ -480,6 +524,20 @@ function ChartSeriesEditorComponent({ )} + {tableSource?.kind === SourceKind.Metric && metricName && ( + + )} ); } diff --git a/packages/app/src/components/MetricAttributeHelperPanel.tsx b/packages/app/src/components/MetricAttributeHelperPanel.tsx new file mode 100644 index 0000000000..ded944aecf --- /dev/null +++ b/packages/app/src/components/MetricAttributeHelperPanel.tsx @@ -0,0 +1,493 @@ +import { useCallback, useMemo, useState } from 'react'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { + Badge, + Box, + Button, + Collapse, + Flex, + Group, + Loader, + Paper, + ScrollArea, + Stack, + Text, + TextInput, + UnstyledButton, +} from '@mantine/core'; +import { useDebouncedValue, useDisclosure } from '@mantine/hooks'; +import { + IconChevronDown, + IconChevronLeft, + IconChevronRight, + IconFilter, + IconPlus, + IconSearch, +} from '@tabler/icons-react'; + +import { useFetchMetricAttributeValues } from '@/hooks/useFetchMetricAttributeValues'; +import { MetricMetadata } from '@/hooks/useFetchMetricMetadata'; +import { + AttributeCategory, + AttributeKey, +} from '@/hooks/useFetchMetricResourceAttrs'; + +interface MetricAttributeHelperPanelProps { + databaseName: string; + metricType: string; + metricName: string; + tableSource: TSource | undefined; + attributeKeys: AttributeKey[]; + isLoading?: boolean; + language: 'sql' | 'lucene'; + metricMetadata?: MetricMetadata | null; + onAddToWhere: (clause: string) => void; + onAddToGroupBy: (clause: string) => void; +} + +const CATEGORY_LABELS: Record = { + ResourceAttributes: 'Resource', + Attributes: 'Attributes', + ScopeAttributes: 'Scope', +}; + +// UCUM (Unified Code for Units of Measure) case-sensitive codes to human-readable names +// Reference: https://ucum.org/ucum +const UCUM_UNIT_NAMES: Record = { + // Time units + s: 'Seconds', + ms: 'Milliseconds', + us: 'Microseconds', + ns: 'Nanoseconds', + min: 'Minutes', + h: 'Hours', + d: 'Days', + wk: 'Weeks', + mo: 'Months', + a: 'Years', + // Data units + By: 'Bytes', + KiBy: 'Kibibytes', + MiBy: 'Mebibytes', + GiBy: 'Gibibytes', + TiBy: 'Tebibytes', + kBy: 'Kilobytes', + MBy: 'Megabytes', + GBy: 'Gigabytes', + TBy: 'Terabytes', + bit: 'Bits', + Kibit: 'Kibibits', + Mibit: 'Mebibits', + Gibit: 'Gibibits', + // Frequency + Hz: 'Hertz', + kHz: 'Kilohertz', + MHz: 'Megahertz', + GHz: 'Gigahertz', + // Temperature + Cel: 'Celsius', + K: 'Kelvin', + '[degF]': 'Fahrenheit', + // Percentage and dimensionless + '%': 'Percent', + '1': 'Count', + '{request}': 'Requests', + '{connection}': 'Connections', + '{error}': 'Errors', + '{packet}': 'Packets', + '{thread}': 'Threads', + '{process}': 'Processes', + '{message}': 'Messages', + '{operation}': 'Operations', + '{call}': 'Calls', + '{fault}': 'Faults', + '{cpu}': 'CPUs', + // Physical units + m: 'Meters', + km: 'Kilometers', + g: 'Grams', + kg: 'Kilograms', + A: 'Amperes', + V: 'Volts', + W: 'Watts', + kW: 'Kilowatts', + J: 'Joules', + kJ: 'Kilojoules', +}; + +// Format UCUM unit code to human-readable name +function formatUnitDisplay(unit: string): string { + // Direct match + if (UCUM_UNIT_NAMES[unit]) { + return UCUM_UNIT_NAMES[unit]; + } + + // Handle compound units like "By/s" -> "Bytes/Second" + if (unit.includes('/')) { + const [numerator, denominator] = unit.split('/'); + const numName = UCUM_UNIT_NAMES[numerator] || numerator; + const denomName = UCUM_UNIT_NAMES[denominator] || denominator; + // Singularize denominator (remove trailing 's' if present) + const singularDenom = denomName.endsWith('s') + ? denomName.slice(0, -1) + : denomName; + return `${numName}/${singularDenom}`; + } + + // Return original if no mapping found + return unit; +} + +function formatWhereClause( + category: AttributeCategory, + name: string, + value: string, + language: 'sql' | 'lucene', +): string { + if (language === 'sql') { + return `${category}['${name}'] = '${value}'`; + } + return `${category}.${name}:"${value}"`; +} + +function formatGroupByClause( + category: AttributeCategory, + name: string, + language: 'sql' | 'lucene', +): string { + if (language === 'sql') { + return `${category}['${name}']`; + } + return `${category}.${name}`; +} + +interface AttributeValueListProps { + databaseName: string; + metricType: string; + metricName: string; + tableSource: TSource | undefined; + attribute: AttributeKey; + language: 'sql' | 'lucene'; + onAddToWhere: (clause: string) => void; + onBack: () => void; + onAddToGroupBy: (clause: string) => void; +} + +function AttributeValueList({ + databaseName, + metricType, + metricName, + tableSource, + attribute, + language, + onAddToWhere, + onBack, + onAddToGroupBy, +}: AttributeValueListProps) { + const [searchTerm, setSearchTerm] = useState(''); + const [debouncedSearch] = useDebouncedValue(searchTerm, 300); + + const { data: values, isLoading } = useFetchMetricAttributeValues({ + databaseName, + metricType, + metricName, + attributeName: attribute.name, + attributeCategory: attribute.category, + searchTerm: debouncedSearch, + tableSource, + }); + + const handleAddValueToWhere = useCallback( + (value: string) => { + const clause = formatWhereClause( + attribute.category, + attribute.name, + value, + language, + ); + onAddToWhere(clause); + }, + [attribute, language, onAddToWhere], + ); + + const handleAddToGroupBy = useCallback(() => { + // Group By is always SQL syntax, regardless of Where condition language + const clause = formatGroupByClause( + attribute.category, + attribute.name, + 'sql', + ); + onAddToGroupBy(clause); + }, [attribute, onAddToGroupBy]); + + return ( + + + + + + + {attribute.name} + + + {CATEGORY_LABELS[attribute.category]} + + + + + + + } + value={searchTerm} + onChange={e => setSearchTerm(e.currentTarget.value)} + /> + + {isLoading ? ( + + + + ) : values && values.length > 0 ? ( + + + {values.map(value => ( + + + {value} + + + + ))} + + + ) : ( + + {searchTerm ? 'No matching values found' : 'No values found'} + + )} + + ); +} + +interface AttributeListProps { + attributeKeys: AttributeKey[]; + onSelectAttribute: (attr: AttributeKey) => void; +} + +function AttributeList({ + attributeKeys, + onSelectAttribute, +}: AttributeListProps) { + const [searchTerm, setSearchTerm] = useState(''); + + const filteredAttributes = useMemo(() => { + if (!searchTerm) return attributeKeys; + const lower = searchTerm.toLowerCase(); + return attributeKeys.filter(attr => + attr.name.toLowerCase().includes(lower), + ); + }, [attributeKeys, searchTerm]); + + const groupedAttributes = useMemo(() => { + const groups: Record = { + ResourceAttributes: [], + Attributes: [], + ScopeAttributes: [], + }; + for (const attr of filteredAttributes) { + groups[attr.category].push(attr); + } + return groups; + }, [filteredAttributes]); + + const categories: AttributeCategory[] = [ + 'ResourceAttributes', + 'Attributes', + 'ScopeAttributes', + ]; + + return ( + + } + value={searchTerm} + onChange={e => setSearchTerm(e.currentTarget.value)} + /> + + + + {categories.map(category => { + const attrs = groupedAttributes[category]; + if (attrs.length === 0) return null; + + return ( + + + + {CATEGORY_LABELS[category]} + + + {attrs.length} + + + + {attrs.map(attr => ( + onSelectAttribute(attr)} + > + + + {attr.name} + + + + + ))} + + + ); + })} + + + + ); +} + +export function MetricAttributeHelperPanel({ + databaseName, + metricName, + tableSource, + metricType, + attributeKeys, + isLoading, + language, + metricMetadata, + onAddToWhere, + onAddToGroupBy, +}: MetricAttributeHelperPanelProps) { + const [opened, { toggle }] = useDisclosure(false); + const [selectedAttribute, setSelectedAttribute] = + useState(null); + + const handleSelectAttribute = useCallback((attr: AttributeKey) => { + setSelectedAttribute(attr); + }, []); + + const handleBack = useCallback(() => { + setSelectedAttribute(null); + }, []); + + if (!metricName) { + return null; + } + + return ( + + + + + {metricMetadata?.description ? ( + + {metricMetadata.description} + + ) : ( + + {metricName} + + )} + {metricMetadata?.unit && ( + + + Unit: + + + {formatUnitDisplay(metricMetadata.unit)} + + + )} + + + {attributeKeys.length > 0 && ( + + {attributeKeys.length} attributes + + )} + + + + + + + + {isLoading ? ( + + + + ) : attributeKeys.length === 0 ? ( + + No attributes found for this metric + + ) : selectedAttribute ? ( + + ) : ( + + )} + + + + ); +} diff --git a/packages/app/src/components/__tests__/DBEditTimeChartForm.test.tsx b/packages/app/src/components/__tests__/DBEditTimeChartForm.test.tsx index 072943a52b..3c33ec957b 100644 --- a/packages/app/src/components/__tests__/DBEditTimeChartForm.test.tsx +++ b/packages/app/src/components/__tests__/DBEditTimeChartForm.test.tsx @@ -18,6 +18,13 @@ jest.mock('@/hooks/useFetchMetricResourceAttrs', () => ({ useFetchMetricResourceAttrs: jest.fn().mockReturnValue({ data: [], }), + parseAttributeKeysFromSuggestions: jest.fn().mockReturnValue([]), +})); + +jest.mock('@/hooks/useFetchMetricMetadata', () => ({ + useFetchMetricMetadata: jest.fn().mockReturnValue({ + data: null, + }), })); jest.mock('@/hooks/useMetadata', () => ({ diff --git a/packages/app/src/hooks/useFetchMetricAttributeValues.tsx b/packages/app/src/hooks/useFetchMetricAttributeValues.tsx new file mode 100644 index 0000000000..2d010d588c --- /dev/null +++ b/packages/app/src/hooks/useFetchMetricAttributeValues.tsx @@ -0,0 +1,112 @@ +import { + chSql, + ResponseJSON, + tableExpr, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { useQuery } from '@tanstack/react-query'; + +import { getClickhouseClient } from '@/clickhouse'; +import { getMetricTableName } from '@/utils'; + +import { AttributeCategory } from './useFetchMetricResourceAttrs'; + +const ATTRIBUTE_VALUES_LIMIT = 100; + +interface MetricAttributeValuesProps { + databaseName: string; + metricName: string; + attributeName: string; + attributeCategory: AttributeCategory; + searchTerm?: string; + tableSource: TSource | undefined; + metricType: string; + enabled?: boolean; +} + +interface AttributeValueResponse { + value: string; +} + +export const useFetchMetricAttributeValues = ({ + databaseName, + metricType, + metricName, + attributeName, + attributeCategory, + searchTerm, + tableSource, + enabled = true, +}: MetricAttributeValuesProps) => { + const tableName = tableSource + ? (getMetricTableName(tableSource, metricType) ?? '') + : ''; + + const shouldFetch = Boolean( + enabled && + databaseName && + tableName && + metricType && + metricName && + attributeName && + attributeCategory && + tableSource && + tableSource?.kind === SourceKind.Metric, + ); + + return useQuery({ + queryKey: [ + 'metric-attribute-values', + metricName, + metricType, + attributeName, + attributeCategory, + searchTerm, + tableSource, + ], + queryFn: async ({ signal }) => { + if (!shouldFetch) { + return []; + } + + const clickhouseClient = getClickhouseClient(); + + // Build optional search filter + const searchFilter = searchTerm + ? chSql` AND ${attributeCategory}[${{ String: attributeName }}] ILIKE ${{ String: `%${searchTerm}%` }}` + : chSql``; + + const sql = chSql` + SELECT DISTINCT ${attributeCategory}[${{ String: attributeName }}] as value + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE MetricName = ${{ String: metricName }} + AND ${attributeCategory}[${{ String: attributeName }}] != '' + ${searchFilter} + ORDER BY value + LIMIT ${{ Int32: ATTRIBUTE_VALUES_LIMIT }} + `; + + const result = (await clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + abort_signal: signal, + connectionId: tableSource!.connection, + clickhouse_settings: { + max_execution_time: 60, + timeout_overflow_mode: 'break', + }, + }) + .then(res => res.json())) as ResponseJSON; + + if (result?.data) { + return result.data.map(row => row.value).filter(Boolean); + } + + return []; + }, + enabled: shouldFetch, + staleTime: 1000 * 60 * 5, // Cache for 5 minutes + }); +}; diff --git a/packages/app/src/hooks/useFetchMetricMetadata.tsx b/packages/app/src/hooks/useFetchMetricMetadata.tsx new file mode 100644 index 0000000000..2b9149c64a --- /dev/null +++ b/packages/app/src/hooks/useFetchMetricMetadata.tsx @@ -0,0 +1,87 @@ +import { + chSql, + ResponseJSON, + tableExpr, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { useQuery } from '@tanstack/react-query'; + +import { getClickhouseClient } from '@/clickhouse'; +import { getMetricTableName } from '@/utils'; + +export interface MetricMetadata { + unit: string; + description: string; +} + +interface MetricMetadataProps { + databaseName: string; + metricType: string; + metricName: string; + tableSource: TSource | undefined; +} + +interface MetricMetadataResponse { + MetricUnit: string; + MetricDescription: string; +} + +export const useFetchMetricMetadata = ({ + databaseName, + metricType, + metricName, + tableSource, +}: MetricMetadataProps) => { + const tableName = tableSource + ? (getMetricTableName(tableSource, metricType) ?? '') + : ''; + + const shouldFetch = Boolean( + databaseName && + metricType && + metricName && + tableSource && + tableName && + tableSource?.kind === SourceKind.Metric, + ); + + return useQuery({ + queryKey: ['metric-metadata', databaseName, metricType, metricName], + queryFn: async ({ signal }) => { + if (!shouldFetch) { + return null; + } + + const clickhouseClient = getClickhouseClient(); + const sql = chSql` + SELECT + MetricUnit, + MetricDescription + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE MetricName = ${{ String: metricName }} + LIMIT 1 + `; + + const result = (await clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + format: 'JSON', + abort_signal: signal, + connectionId: tableSource!.connection, + }) + .then(res => res.json())) as ResponseJSON; + + if (result?.data?.[0]) { + return { + unit: result.data[0].MetricUnit || '', + description: result.data[0].MetricDescription || '', + }; + } + + return null; + }, + enabled: shouldFetch, + staleTime: 1000 * 60 * 5, // Cache for 5 minutes + }); +}; diff --git a/packages/app/src/hooks/useFetchMetricResourceAttrs.tsx b/packages/app/src/hooks/useFetchMetricResourceAttrs.tsx index 6de2589e1d..d591aad7eb 100644 --- a/packages/app/src/hooks/useFetchMetricResourceAttrs.tsx +++ b/packages/app/src/hooks/useFetchMetricResourceAttrs.tsx @@ -8,10 +8,79 @@ import { TSource } from '@hyperdx/common-utils/dist/types'; import { useQuery } from '@tanstack/react-query'; import { getClickhouseClient } from '@/clickhouse'; -import { formatAttributeClause } from '@/utils'; +import { formatAttributeClause, getMetricTableName } from '@/utils'; const METRIC_FETCH_LIMIT = 10000; +export type AttributeCategory = + | 'ResourceAttributes' + | 'ScopeAttributes' + | 'Attributes'; + +export interface AttributeKey { + name: string; + category: AttributeCategory; +} + +// Parse suggestion strings to extract unique attribute keys +// SQL format: ResourceAttributes['key']='value' +// Lucene format: ResourceAttributes.key:"value" +export const parseAttributeKeysFromSuggestions = ( + suggestions: string[], +): AttributeKey[] => { + const categories: AttributeCategory[] = [ + 'ResourceAttributes', + 'ScopeAttributes', + 'Attributes', + ]; + const seen = new Set(); + const attributeKeys: AttributeKey[] = []; + + for (const suggestion of suggestions) { + for (const category of categories) { + if (!suggestion.startsWith(category)) continue; + + let name: string | null = null; + + // Try SQL format: Category['key'] + const sqlMatch = suggestion.match( + new RegExp(`^${category}\\['([^']+)'\\]`), + ); + if (sqlMatch) { + name = sqlMatch[1]; + } else { + // Try Lucene format: Category.key: + const luceneMatch = suggestion.match( + new RegExp(`^${category}\\.([^:]+):`), + ); + if (luceneMatch) { + name = luceneMatch[1]; + } + } + + if (name) { + const uniqueKey = `${category}:${name}`; + if (!seen.has(uniqueKey)) { + seen.add(uniqueKey); + attributeKeys.push({ name, category }); + } + } + break; + } + } + + // Sort by category then name + attributeKeys.sort((a, b) => { + if (a.category !== b.category) { + const order = ['ResourceAttributes', 'Attributes', 'ScopeAttributes']; + return order.indexOf(a.category) - order.indexOf(b.category); + } + return a.name.localeCompare(b.name); + }); + + return attributeKeys; +}; + const extractAttributeKeys = ( attributesArr: MetricAttributesResponse[], isSql: boolean, @@ -52,14 +121,13 @@ const extractAttributeKeys = ( } return Array.from(resultSet); } catch (e) { - console.error('Error parsing metric autocompleteattributes', e); + console.error('Error parsing metric autocomplete attributes', e); return []; } }; interface MetricResourceAttrsProps { databaseName: string; - tableName: string; metricType: string; metricName: string; tableSource: TSource | undefined; @@ -74,28 +142,25 @@ interface MetricAttributesResponse { export const useFetchMetricResourceAttrs = ({ databaseName, - tableName, metricType, metricName, tableSource, isSql, }: MetricResourceAttrsProps) => { + const tableName = tableSource + ? (getMetricTableName(tableSource, metricType) ?? '') + : ''; + const shouldFetch = Boolean( databaseName && tableName && + metricType && tableSource && tableSource?.kind === SourceKind.Metric, ); return useQuery({ - queryKey: [ - 'metric-attributes', - databaseName, - tableName, - metricType, - metricName, - isSql, - ], + queryKey: ['metric-attributes', metricType, metricName, isSql, tableSource], queryFn: async ({ signal }) => { if (!shouldFetch) { return [];