(fix): add support for JSON columns in the service dashboard

Perseus · Perseus · commit bd344b3e6dc7 · 2025-07-26T08:45:42.000Z
diff --git a/packages/app/src/ServicesDashboardPage.tsx b/packages/app/src/ServicesDashboardPage.tsx
@@ -43,6 +43,7 @@ import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor';
 import { TimePicker } from '@/components/TimePicker';
 import WhereLanguageControlled from '@/components/WhereLanguageControlled';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { withAppNav } from '@/layout';
 import SearchInputV2 from '@/SearchInputV2';
 import { getExpressions } from '@/serviceDashboard';
@@ -90,7 +91,12 @@ function ServiceSelectControlled({
   onCreate?: () => void;
 } & UseControllerProps<any>) {
   const { data: source } = useSource({ id: sourceId });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const queriedConfig = {
     ...source,
@@ -153,7 +159,12 @@ export function EndpointLatencyChart({
   appliedConfig?: AppliedConfig;
   extraFilters?: Filter[];
 }) {
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
   const [latencyChartType, setLatencyChartType] = useState<
     'line' | 'histogram'
   >('line');
@@ -259,7 +270,12 @@ function HttpTab({
   appliedConfig: AppliedConfig;
 }) {
   const { data: source } = useSource({ id: appliedConfig.source });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [reqChartType, setReqChartType] = useQueryState(
     'reqChartType',
@@ -529,7 +545,12 @@ function DatabaseTab({
   appliedConfig: AppliedConfig;
 }) {
   const { data: source } = useSource({ id: appliedConfig.source });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [chartType, setChartType] = useState<'table' | 'list'>('list');
 
@@ -776,7 +797,12 @@ function ErrorsTab({
   appliedConfig: AppliedConfig;
 }) {
   const { data: source } = useSource({ id: appliedConfig.source });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   return (
     <Grid mt="md" grow={false} w="100%" maw="100%" overflow="hidden">
diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts
@@ -0,0 +1,68 @@
+import type { TSource } from '@hyperdx/common-utils/dist/types';
+import { SourceKind } from '@hyperdx/common-utils/dist/types';
+
+import { getExpressions } from '../serviceDashboard';
+
+describe('Service Dashboard', () => {
+  const mockSource: TSource = {
+    id: 'test-source',
+    name: 'Test Source',
+    kind: SourceKind.Trace,
+    from: {
+      databaseName: 'test_db',
+      tableName: 'otel_traces_json',
+    },
+    connection: 'test-connection',
+    timestampValueExpression: 'Timestamp',
+    durationExpression: 'Duration',
+    durationPrecision: 9,
+    traceIdExpression: 'TraceId',
+    serviceNameExpression: 'ServiceName',
+    spanNameExpression: 'SpanName',
+    spanKindExpression: 'SpanKind',
+    severityTextExpression: 'StatusCode',
+  };
+
+  describe('getExpressions', () => {
+    it('should use map syntax for non-JSON columns by default', () => {
+      const expressions = getExpressions(mockSource, []);
+
+      expect(expressions.k8sResourceName).toBe(
+        "SpanAttributes['k8s.resource.name']",
+      );
+      expect(expressions.k8sPodName).toBe("SpanAttributes['k8s.pod.name']");
+      expect(expressions.httpScheme).toBe("SpanAttributes['http.scheme']");
+      expect(expressions.serverAddress).toBe(
+        "SpanAttributes['server.address']",
+      );
+      expect(expressions.httpHost).toBe("SpanAttributes['http.host']");
+      expect(expressions.dbStatement).toBe(
+        "coalesce(nullif(SpanAttributes['db.query.text'], ''), nullif(SpanAttributes['db.statement'], ''))",
+      );
+    });
+
+    it('should use backtick syntax when SpanAttributes is a JSON column', () => {
+      const expressions = getExpressions(mockSource, ['SpanAttributes']);
+
+      expect(expressions.k8sResourceName).toBe(
+        'SpanAttributes.`k8s.resource.name`',
+      );
+      expect(expressions.k8sPodName).toBe('SpanAttributes.`k8s.pod.name`');
+      expect(expressions.httpScheme).toBe('SpanAttributes.`http.scheme`');
+      expect(expressions.serverAddress).toBe('SpanAttributes.`server.address`');
+      expect(expressions.httpHost).toBe('SpanAttributes.`http.host`');
+      expect(expressions.dbStatement).toBe(
+        "coalesce(nullif(SpanAttributes.`db.query.text`, ''), nullif(SpanAttributes.`db.statement`, ''))",
+      );
+    });
+
+    it('should work with empty jsonColumns array', () => {
+      const expressions = getExpressions(mockSource);
+
+      // Should default to map syntax
+      expect(expressions.k8sResourceName).toBe(
+        "SpanAttributes['k8s.resource.name']",
+      );
+    });
+  });
+});
diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx
@@ -9,6 +9,7 @@ import { ChartBox } from '@/components/ChartBox';
 import { DBTimeChart } from '@/components/DBTimeChart';
 import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
 import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 import { useSource } from '@/source';
 import { useZIndex, ZIndexContext } from '@/zIndex';
@@ -26,7 +27,12 @@ export default function ServiceDashboardDbQuerySidePanel({
   searchedTimeRange: [Date, Date];
 }) {
   const { data: source } = useSource({ id: sourceId });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [dbQuery, setDbQuery] = useQueryState('dbquery', parseAsString);
   const onClose = useCallback(() => {
diff --git a/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx b/packages/app/src/components/ServiceDashboardEndpointPerformanceChart.tsx
@@ -4,6 +4,7 @@ import { Group, Text } from '@mantine/core';
 import { MS_NUMBER_FORMAT } from '@/ChartUtils';
 import { ChartBox } from '@/components/ChartBox';
 import DBListBarChart from '@/components/DBListBarChart';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 
 const MAX_NUM_GROUPS = 200;
@@ -19,7 +20,12 @@ export default function ServiceDashboardEndpointPerformanceChart({
   service?: string;
   endpoint?: string;
 }) {
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   if (!source) {
     return null;
@@ -42,6 +48,43 @@ export default function ServiceDashboardEndpointPerformanceChart({
     WHERE ${parentSpanWhereCondition}
     `;
 
+  let spanNameColSql = `
+    concat(
+      ${expressions.spanName}, ' ',
+      if(
+        has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}),
+          COALESCE(
+            NULLIF(${expressions.serverAddress}, ''),
+            NULLIF(${expressions.httpHost}, '')
+          ),
+          ''
+    ));`;
+
+  const spanAttributesExpression =
+    source.eventAttributesExpression || 'SpanAttributes';
+
+  // ClickHouse does not support NULLIF(some_dynamic_column)
+  // so we instead use toString() and an empty string check to check for
+  // existence of the serverAddress/httpHost to build the span name
+  if (jsonColumns.includes(spanAttributesExpression)) {
+    spanNameColSql = `
+    concat(
+      ${expressions.spanName}, ' ',
+      if(
+        has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}),
+        if(
+            toString(${expressions.serverAddress}) != '',
+            toString(${expressions.serverAddress}),
+            if(
+              toString(${expressions.httpHost}) != '', 
+              toString(${expressions.httpHost}), 
+              ''
+            )
+          ),
+        ''
+    ))`;
+  }
+
   return (
     <ChartBox style={{ height: 350, overflow: 'auto' }}>
       <Group justify="space-between" align="center" mb="sm">
@@ -60,16 +103,7 @@ export default function ServiceDashboardEndpointPerformanceChart({
             select: [
               {
                 alias: 'group',
-                valueExpression: `concat(
-                  ${expressions.spanName}, ' ',
-                  if(
-                    has(['HTTP DELETE', 'DELETE', 'HTTP GET', 'GET', 'HTTP HEAD', 'HEAD', 'HTTP OPTIONS', 'OPTIONS', 'HTTP PATCH', 'PATCH', 'HTTP POST', 'POST', 'HTTP PUT', 'PUT'], ${expressions.spanName}),
-                    COALESCE(
-                      NULLIF(${expressions.serverAddress}, ''),
-                      NULLIF(${expressions.httpHost}, '')
-                    ),
-                    ''
-                  ))`,
+                valueExpression: spanNameColSql,
               },
               {
                 alias: 'Total Time Spent',
diff --git a/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx b/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx
@@ -13,6 +13,7 @@ import { DBTimeChart } from '@/components/DBTimeChart';
 import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
 import ServiceDashboardEndpointPerformanceChart from '@/components/ServiceDashboardEndpointPerformanceChart';
 import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 import { EndpointLatencyChart } from '@/ServicesDashboardPage';
 import { useSource } from '@/source';
@@ -31,7 +32,12 @@ export default function ServiceDashboardEndpointSidePanel({
   searchedTimeRange: [Date, Date];
 }) {
   const { data: source } = useSource({ id: sourceId });
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [endpoint, setEndpoint] = useQueryState('endpoint', parseAsString);
   const onClose = useCallback(() => {
diff --git a/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx b/packages/app/src/components/ServiceDashboardSlowestEventsTile.tsx
@@ -8,6 +8,7 @@ import { ChartBox } from '@/components/ChartBox';
 import DBRowSidePanel from '@/components/DBRowSidePanel';
 import { DBSqlRowTable } from '@/components/DBRowTable';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
+import { useJsonColumns } from '@/hooks/useMetadata';
 import { getExpressions } from '@/serviceDashboard';
 import { useSource } from '@/source';
 
@@ -30,7 +31,12 @@ export default function SlowestEventsTile({
   enabled?: boolean;
   extraFilters?: Filter[];
 }) {
-  const expressions = getExpressions(source);
+  const { data: jsonColumns = [] } = useJsonColumns({
+    databaseName: source?.from?.databaseName || '',
+    tableName: source?.from?.tableName || '',
+    connectionId: source?.connection || '',
+  });
+  const expressions = getExpressions(source, jsonColumns);
 
   const [rowId, setRowId] = useQueryState('rowId', parseAsString);
   const [rowSource, setRowSource] = useQueryState('rowSource', parseAsString);
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
@@ -42,6 +42,33 @@ export function useColumns(
   });
 }
 
+export function useJsonColumns(
+  {
+    databaseName,
+    tableName,
+    connectionId,
+  }: {
+    databaseName: string;
+    tableName: string;
+    connectionId: string;
+  },
+  options?: Partial<UseQueryOptions<string[]>>,
+) {
+  return useQuery<string[]>({
+    queryKey: ['useMetadata.useJsonColumns', { databaseName, tableName }],
+    queryFn: async () => {
+      const metadata = getMetadata();
+      return metadata.getJsonColumns({
+        databaseName,
+        tableName,
+        connectionId,
+      });
+    },
+    enabled: !!databaseName && !!tableName && !!connectionId,
+    ...options,
+  });
+}
+
 export function useAllFields(
   _tableConnections: TableConnection | TableConnection[],
   options?: Partial<UseQueryOptions<Field[]>>,
diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts
@@ -1,7 +1,17 @@
 import { TSource } from '@hyperdx/common-utils/dist/types';
 
-function getDefaults() {
+function getDefaults(jsonColumns: string[] = []) {
   const spanAttributeField = 'SpanAttributes';
+  const isJsonColumn = jsonColumns.includes(spanAttributeField);
+
+  // Helper function to format field access based on column type
+  const formatFieldAccess = (field: string, key: string) => {
+    if (isJsonColumn) {
+      return `${field}.\`${key}\``;
+    } else {
+      return `${field}['${key}']`;
+    }
+  };
 
   return {
     duration: 'Duration',
@@ -11,17 +21,17 @@ function getDefaults() {
     spanName: 'SpanName',
     spanKind: 'SpanKind',
     severityText: 'StatusCode',
-    k8sResourceName: `${spanAttributeField}['k8s.resource.name']`,
-    k8sPodName: `${spanAttributeField}['k8s.pod.name']`,
-    httpScheme: `${spanAttributeField}['http.scheme']`,
-    serverAddress: `${spanAttributeField}['server.address']`,
-    httpHost: `${spanAttributeField}['http.host']`,
-    dbStatement: `coalesce(nullif(${spanAttributeField}['db.query.text'], ''), nullif(${spanAttributeField}['db.statement'], ''))`,
+    k8sResourceName: formatFieldAccess(spanAttributeField, 'k8s.resource.name'),
+    k8sPodName: formatFieldAccess(spanAttributeField, 'k8s.pod.name'),
+    httpScheme: formatFieldAccess(spanAttributeField, 'http.scheme'),
+    serverAddress: formatFieldAccess(spanAttributeField, 'server.address'),
+    httpHost: formatFieldAccess(spanAttributeField, 'http.host'),
+    dbStatement: `coalesce(nullif(${formatFieldAccess(spanAttributeField, 'db.query.text')}, ''), nullif(${formatFieldAccess(spanAttributeField, 'db.statement')}, ''))`,
   };
 }
 
-export function getExpressions(source?: TSource) {
-  const defaults = getDefaults();
+export function getExpressions(source?: TSource, jsonColumns: string[] = []) {
+  const defaults = getDefaults(jsonColumns);
 
   const fieldExpressions = {
     // General
diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts
