From fc379e8541c898187f9bc08beba092c7492e61c1 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Mon, 29 Sep 2025 17:29:35 -0400 Subject: [PATCH 1/3] Revert "fix: disable json metadata field and key fetching (#1200)" This reverts commit 816f90a3923995bf38c9e57f9682ce79b2db2e98. --- .changeset/orange-mayflies-lie.md | 5 ----- packages/common-utils/src/metadata.ts | 4 ---- 2 files changed, 9 deletions(-) delete mode 100644 .changeset/orange-mayflies-lie.md diff --git a/.changeset/orange-mayflies-lie.md b/.changeset/orange-mayflies-lie.md deleted file mode 100644 index 37667abb0..000000000 --- a/.changeset/orange-mayflies-lie.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix: disable json filters for now diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index efcb88391..23d08dc50 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -347,8 +347,6 @@ export class Metadata { column: string; maxKeys?: number; } & TableConnection) { - // HDX-2480 delete line below to reenable json filters - return []; // Need to disable JSON keys for the time being. const cacheKey = metricName ? `${databaseName}.${tableName}.${column}.${metricName}.keys` : `${databaseName}.${tableName}.${column}.keys`; @@ -487,8 +485,6 @@ export class Metadata { }); for (const c of columns) { - // HDX-2480 delete condition below to reenable json filters - if (c.type === 'JSON') continue; fields.push({ path: [c.name], type: c.type, From 1992c3d2ef31cf91d453340298c0bd3b231c3077 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 30 Sep 2025 12:40:52 -0400 Subject: [PATCH 2/3] query from JSON Keys columns --- .../src/components/DBSearchPageFilters.tsx | 7 +- packages/common-utils/src/metadata.ts | 82 ++++++++++--------- 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index a15689dff..156b9b5bc 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -666,7 +666,12 @@ const DBSearchPageFiltersComponent = ({ ...chartConfig, dateRange, }, - keys: [key], + keys: [ + key + .split('.') + .map(v => `\`${v}\``) + .join('.'), + ], limit: 200, disableRowLimit: true, }); diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index 23d08dc50..00c4bb705 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -357,47 +357,51 @@ export class Metadata { const where = metricName ? chSql`WHERE MetricName=${{ String: metricName }}` : ''; - const sql = chSql`WITH all_paths AS - ( - SELECT DISTINCT JSONDynamicPathsWithTypes(${{ Identifier: column }}) as paths - FROM ${tableExpr({ database: databaseName, table: tableName })} ${where} - LIMIT ${{ Int32: maxKeys }} - SETTINGS timeout_overflow_mode = 'break', max_execution_time = 2 - ) - SELECT groupUniqArrayMap(paths) as pathMap - FROM all_paths;`; - const keys = await this.clickhouseClient - .query<'JSON'>({ - query: sql.sql, - query_params: sql.params, - connectionId, - clickhouse_settings: { - max_rows_to_read: String( - this.getClickHouseSettings().max_rows_to_read ?? - DEFAULT_METADATA_MAX_ROWS_TO_READ, - ), - read_overflow_mode: 'break', - ...this.getClickHouseSettings(), - }, - }) - .then(res => res.json<{ pathMap: Record }>()) - .then(d => { - const keys: { key: string; chType: string }[] = []; - for (const [key, typeArr] of Object.entries(d.data[0].pathMap)) { - if (!key || !typeArr || !Array.isArray(typeArr)) { - throw new Error( - `Error fetching keys for filters (key: ${key}, typeArr: ${typeArr})`, - ); + try { + // First try to use the Paths column + const sql = chSql`SELECT DISTINCT arrayJoin(${{ Identifier: `${column}Keys` }}) as path + FROM ${tableExpr({ database: databaseName, table: tableName })} ${where} + LIMIT ${{ Int32: maxKeys }} + SETTINGS timeout_overflow_mode = 'break', max_execution_time = 2`; + + const keys = await this.clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + max_rows_to_read: String( + this.getClickHouseSettings().max_rows_to_read ?? + DEFAULT_METADATA_MAX_ROWS_TO_READ, + ), + read_overflow_mode: 'break', + ...this.getClickHouseSettings(), + }, + }) + .then(res => res.json<{ path: string }>()) + .then(d => { + const keys: { key: string; chType: string }[] = []; + for (const row of d.data) { + if (!row.path) { + continue; + } + keys.push({ + key: row.path, + chType: 'String', // Default to String type for path entries + }); } - keys.push({ - key: key, - chType: typeArr[0], - }); - } - return keys; - }); - return keys; + return keys; + }); + return keys; + } catch (error) { + // If the Paths column doesn't exist, return empty array + console.warn( + `Column ${column}Paths not found in table ${databaseName}.${tableName}:`, + error, + ); + return []; + } }, ); } From d2a8dd7355aad3359b58a0c7ef640cfe5571c561 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 30 Sep 2025 13:10:17 -0400 Subject: [PATCH 3/3] optimize return format --- packages/common-utils/src/metadata.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index 00c4bb705..f448d5626 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -360,16 +360,17 @@ export class Metadata { try { // First try to use the Paths column - const sql = chSql`SELECT DISTINCT arrayJoin(${{ Identifier: `${column}Keys` }}) as path + const sql = chSql`SELECT DISTINCT arrayJoin(${{ Identifier: `${column}Keys` }}) as keys FROM ${tableExpr({ database: databaseName, table: tableName })} ${where} LIMIT ${{ Int32: maxKeys }} SETTINGS timeout_overflow_mode = 'break', max_execution_time = 2`; const keys = await this.clickhouseClient - .query<'JSON'>({ + .query({ query: sql.sql, query_params: sql.params, connectionId, + format: 'JSONColumnsWithMetadata', clickhouse_settings: { max_rows_to_read: String( this.getClickHouseSettings().max_rows_to_read ?? @@ -379,15 +380,16 @@ export class Metadata { ...this.getClickHouseSettings(), }, }) - .then(res => res.json<{ path: string }>()) + .then(res => res.json()) .then(d => { const keys: { key: string; chType: string }[] = []; - for (const row of d.data) { - if (!row.path) { + for (const key of (d.data as unknown as { keys: string[] }) + .keys) { + if (!key) { continue; } keys.push({ - key: row.path, + key: key, chType: 'String', // Default to String type for path entries }); }