diff --git a/sqlmesh/lsp/api.py b/sqlmesh/lsp/api.py index a034283759..3135614d4b 100644 --- a/sqlmesh/lsp/api.py +++ b/sqlmesh/lsp/api.py @@ -13,7 +13,7 @@ CustomMethodRequestBaseClass, CustomMethodResponseBaseClass, ) -from web.server.models import LineageColumn, Model +from web.server.models import LineageColumn, Model, TableDiff API_FEATURE = "sqlmesh/api" @@ -25,7 +25,7 @@ class ApiRequest(CustomMethodRequestBaseClass): """ requestId: str - url: str + endpoint: str method: t.Optional[str] = "GET" params: t.Optional[t.Dict[str, t.Any]] = None body: t.Optional[t.Dict[str, t.Any]] = None @@ -74,3 +74,11 @@ class ApiResponseGetColumnLineage(BaseAPIResponse): """ data: t.Dict[str, t.Dict[str, LineageColumn]] + + +class ApiResponseGetTableDiff(BaseAPIResponse): + """ + Response from the SQLMesh API for the get_table_diff endpoint. + """ + + data: t.Optional[TableDiff] diff --git a/sqlmesh/lsp/custom.py b/sqlmesh/lsp/custom.py index 8ad6418401..84be43ee0e 100644 --- a/sqlmesh/lsp/custom.py +++ b/sqlmesh/lsp/custom.py @@ -158,6 +158,17 @@ class ListWorkspaceTestsRequest(CustomMethodRequestBaseClass): pass +GET_ENVIRONMENTS_FEATURE = "sqlmesh/get_environments" + + +class GetEnvironmentsRequest(CustomMethodRequestBaseClass): + """ + Request to get all environments in the current project. + """ + + pass + + class TestEntry(PydanticModel): """ An entry representing a test in the workspace. @@ -194,3 +205,53 @@ class RunTestRequest(CustomMethodRequestBaseClass): class RunTestResponse(CustomMethodResponseBaseClass): success: bool error_message: t.Optional[str] = None + + +class EnvironmentInfo(PydanticModel): + """ + Information about an environment. + """ + + name: str + snapshots: t.List[str] + start_at: str + plan_id: str + + +class GetEnvironmentsResponse(CustomMethodResponseBaseClass): + """ + Response containing all environments in the current project. + """ + + environments: t.Dict[str, EnvironmentInfo] + pinned_environments: t.Set[str] + default_target_environment: str + + +GET_MODELS_FEATURE = "sqlmesh/get_models" + + +class GetModelsRequest(CustomMethodRequestBaseClass): + """ + Request to get all models available for table diff. + """ + + pass + + +class ModelInfo(PydanticModel): + """ + Information about a model for table diff. + """ + + name: str + fqn: str + description: t.Optional[str] = None + + +class GetModelsResponse(CustomMethodResponseBaseClass): + """ + Response containing all models available for table diff. + """ + + models: t.List[ModelInfo] diff --git a/sqlmesh/lsp/main.py b/sqlmesh/lsp/main.py index c05fff4551..4d91dcc071 100755 --- a/sqlmesh/lsp/main.py +++ b/sqlmesh/lsp/main.py @@ -14,14 +14,17 @@ WorkspaceInlayHintRefreshRequest, ) from pygls.server import LanguageServer +from sqlglot import exp from sqlmesh._version import __version__ from sqlmesh.core.context import Context +from sqlmesh.utils.date import to_timestamp from sqlmesh.lsp.api import ( API_FEATURE, ApiRequest, ApiResponseGetColumnLineage, ApiResponseGetLineage, ApiResponseGetModels, + ApiResponseGetTableDiff, ) from sqlmesh.lsp.commands import EXTERNAL_MODEL_UPDATE_COLUMNS @@ -36,6 +39,8 @@ RENDER_MODEL_FEATURE, SUPPORTED_METHODS_FEATURE, FORMAT_PROJECT_FEATURE, + GET_ENVIRONMENTS_FEATURE, + GET_MODELS_FEATURE, AllModelsRequest, AllModelsResponse, AllModelsForRenderRequest, @@ -57,6 +62,12 @@ RUN_TEST_FEATURE, RunTestRequest, RunTestResponse, + GetEnvironmentsRequest, + GetEnvironmentsResponse, + EnvironmentInfo, + GetModelsRequest, + GetModelsResponse, + ModelInfo, ) from sqlmesh.lsp.errors import ContextFailedError, context_error_to_diagnostic from sqlmesh.lsp.helpers import to_lsp_range, to_sqlmesh_position @@ -74,9 +85,12 @@ from sqlmesh.utils.pydantic import PydanticModel from web.server.api.endpoints.lineage import column_lineage, model_lineage from web.server.api.endpoints.models import get_models +from web.server.api.endpoints.table_diff import _process_sample_data from typing import Union from dataclasses import dataclass, field +from web.server.models import RowDiff, SchemaDiff, TableDiff + class InitializationOptions(PydanticModel): """Initialization options for the SQLMesh Language Server, that @@ -154,6 +168,8 @@ def __init__( LIST_WORKSPACE_TESTS_FEATURE: self._list_workspace_tests, LIST_DOCUMENT_TESTS_FEATURE: self._list_document_tests, RUN_TEST_FEATURE: self._run_test, + GET_ENVIRONMENTS_FEATURE: self._custom_get_environments, + GET_MODELS_FEATURE: self._custom_get_models, } # Register LSP features (e.g., formatting, hover, etc.) @@ -246,13 +262,71 @@ def _custom_format_project( ls.log_trace(f"Error formatting project: {e}") return FormatProjectResponse() + def _custom_get_environments( + self, ls: LanguageServer, params: GetEnvironmentsRequest + ) -> GetEnvironmentsResponse: + """Get all environments in the current project.""" + try: + context = self._context_get_or_load() + environments = {} + + # Get environments from state + for env in context.context.state_reader.get_environments(): + environments[env.name] = EnvironmentInfo( + name=env.name, + snapshots=[s.fingerprint.to_identifier() for s in env.snapshots], + start_at=str(to_timestamp(env.start_at)), + plan_id=env.plan_id or "", + ) + + return GetEnvironmentsResponse( + environments=environments, + pinned_environments=context.context.config.pinned_environments, + default_target_environment=context.context.config.default_target_environment, + ) + except Exception as e: + ls.log_trace(f"Error getting environments: {e}") + return GetEnvironmentsResponse( + response_error=str(e), + environments={}, + pinned_environments=set(), + default_target_environment="", + ) + + def _custom_get_models(self, ls: LanguageServer, params: GetModelsRequest) -> GetModelsResponse: + """Get all models available for table diff.""" + try: + context = self._context_get_or_load() + models = [ + ModelInfo( + name=model.name, + fqn=model.fqn, + description=model.description, + ) + for model in context.context.models.values() + # Filter for models that are suitable for table diff + if model._path is not None # Has a file path + ] + return GetModelsResponse(models=models) + except Exception as e: + ls.log_trace(f"Error getting table diff models: {e}") + return GetModelsResponse( + response_error=str(e), + models=[], + ) + def _custom_api( self, ls: LanguageServer, request: ApiRequest - ) -> t.Union[ApiResponseGetModels, ApiResponseGetColumnLineage, ApiResponseGetLineage]: + ) -> t.Union[ + ApiResponseGetModels, + ApiResponseGetColumnLineage, + ApiResponseGetLineage, + ApiResponseGetTableDiff, + ]: ls.log_trace(f"API request: {request}") context = self._context_get_or_load() - parsed_url = urllib.parse.urlparse(request.url) + parsed_url = urllib.parse.urlparse(request.endpoint) path_parts = parsed_url.path.strip("/").split("/") if request.method == "GET": @@ -280,7 +354,76 @@ def _custom_api( ) return ApiResponseGetColumnLineage(data=column_lineage_response) - raise NotImplementedError(f"API request not implemented: {request.url}") + if path_parts[:2] == ["api", "table_diff"]: + import numpy as np + + # /api/table_diff + params = request.params + table_diff_result: t.Optional[TableDiff] = None + if params := request.params: + source = getattr(params, "source", "") if params else "" + target = getattr(params, "target", "") if params else "" + on = getattr(params, "on", None) if params else None + model_or_snapshot = ( + getattr(params, "model_or_snapshot", None) if params else None + ) + where = getattr(params, "where", None) if params else None + temp_schema = getattr(params, "temp_schema", None) if params else None + limit = getattr(params, "limit", 20) if params else 20 + + table_diffs = context.context.table_diff( + source=source, + target=target, + on=exp.condition(on) if on else None, + select_models={model_or_snapshot} if model_or_snapshot else None, + where=where, + limit=limit, + show=False, + ) + + if table_diffs: + diff = table_diffs[0] if isinstance(table_diffs, list) else table_diffs + + _schema_diff = diff.schema_diff() + _row_diff = diff.row_diff(temp_schema=temp_schema) + schema_diff = SchemaDiff( + source=_schema_diff.source, + target=_schema_diff.target, + source_schema=_schema_diff.source_schema, + target_schema=_schema_diff.target_schema, + added=_schema_diff.added, + removed=_schema_diff.removed, + modified=_schema_diff.modified, + ) + + # create a readable column-centric sample data structure + processed_sample_data = _process_sample_data(_row_diff, source, target) + + row_diff = RowDiff( + source=_row_diff.source, + target=_row_diff.target, + stats=_row_diff.stats, + sample=_row_diff.sample.replace({np.nan: None}).to_dict(), + joined_sample=_row_diff.joined_sample.replace({np.nan: None}).to_dict(), + s_sample=_row_diff.s_sample.replace({np.nan: None}).to_dict(), + t_sample=_row_diff.t_sample.replace({np.nan: None}).to_dict(), + column_stats=_row_diff.column_stats.replace({np.nan: None}).to_dict(), + source_count=_row_diff.source_count, + target_count=_row_diff.target_count, + count_pct_change=_row_diff.count_pct_change, + decimals=getattr(_row_diff, "decimals", 3), + processed_sample_data=processed_sample_data, + ) + + s_index, t_index, _ = diff.key_columns + table_diff_result = TableDiff( + schema_diff=schema_diff, + row_diff=row_diff, + on=[(s.name, t.name) for s, t in zip(s_index, t_index)], + ) + return ApiResponseGetTableDiff(data=table_diff_result) + + raise NotImplementedError(f"API request not implemented: {request.endpoint}") def _custom_supported_methods( self, ls: LanguageServer, params: SupportedMethodsRequest diff --git a/vscode/bus/src/callbacks.ts b/vscode/bus/src/callbacks.ts index 8c492ace8c..180ed0f330 100644 --- a/vscode/bus/src/callbacks.ts +++ b/vscode/bus/src/callbacks.ts @@ -51,13 +51,56 @@ export type RPCMethods = { } api_query: { params: { - url: string + endpoint: string method: string params: any body: any } result: any } + get_selected_model: { + params: {} + result: { + selectedModel?: any + } + } + get_all_models: { + params: {} + result: { + ok: boolean + models?: any[] + error?: string + } + } + set_selected_model: { + params: { + model: any + } + result: { + ok: boolean + selectedModel?: any + } + } + get_environments: { + params: {} + result: { + ok: boolean + environments?: Record + error?: string + } + } + run_table_diff: { + params: { + sourceModel: string + sourceEnvironment: string + targetEnvironment: string + } + result: { + ok: boolean + data?: any + error?: string + } + } } & RPCMethodsShape export type RPCRequest = { diff --git a/vscode/extension/assets/images/diff.svg b/vscode/extension/assets/images/diff.svg new file mode 100644 index 0000000000..fec20deaa1 --- /dev/null +++ b/vscode/extension/assets/images/diff.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/vscode/extension/package.json b/vscode/extension/package.json index 188a91af22..35499ad68f 100644 --- a/vscode/extension/package.json +++ b/vscode/extension/package.json @@ -61,7 +61,8 @@ { "id": "sqlmesh.lineage", "name": "", - "type": "webview" + "type": "webview", + "icon": "./assets/images/dag.svg" } ] }, @@ -112,6 +113,12 @@ "command": "sqlmesh.stop", "title": "SQLMesh: Stop Server", "description": "SQLMesh" + }, + { + "command": "sqlmesh.showTableDiff", + "title": "SQLMesh: Show Table Diff", + "description": "SQLMesh", + "icon": "$(diff)" } ], "menus": { @@ -120,6 +127,11 @@ "command": "sqlmesh.renderModel", "when": "resourceExtname == .sql", "group": "navigation" + }, + { + "command": "sqlmesh.showTableDiff", + "when": "resourceExtname == .sql", + "group": "navigation" } ] } diff --git a/vscode/extension/src/commands/tableDiff.ts b/vscode/extension/src/commands/tableDiff.ts new file mode 100644 index 0000000000..ac1a7a3069 --- /dev/null +++ b/vscode/extension/src/commands/tableDiff.ts @@ -0,0 +1,589 @@ +import * as vscode from 'vscode' +import { LSPClient } from '../lsp/lsp' +import { isErr } from '@bus/result' +import { CallbackEvent, RPCRequest } from '@bus/callbacks' +import { getWorkspaceFolders } from '../utilities/common/vscodeapi' + +interface ModelInfo { + name: string + fqn: string + description?: string | null +} + +export function showTableDiff( + lspClient?: LSPClient, + extensionUri?: vscode.Uri, +) { + return async () => { + if (!lspClient) { + vscode.window.showErrorMessage('LSP client not available') + return + } + + if (!extensionUri) { + vscode.window.showErrorMessage('Extension URI not available') + return + } + + // Get the current active editor + const activeEditor = vscode.window.activeTextEditor + let selectedModelInfo: ModelInfo | null = null + + if (!activeEditor) { + // No active editor, show a list of all models + const allModelsResult = await lspClient.call_custom_method( + 'sqlmesh/get_models', + {}, + ) + + if (isErr(allModelsResult)) { + vscode.window.showErrorMessage( + `Failed to get models: ${allModelsResult.error.message}`, + ) + return + } + + if ( + !allModelsResult.value.models || + allModelsResult.value.models.length === 0 + ) { + vscode.window.showInformationMessage('No models found in the project') + return + } + + // Let user choose from all models + const items = (allModelsResult.value.models as ModelInfo[]).map( + (model: ModelInfo) => ({ + label: model.name, + description: model.fqn, + detail: model.description ? model.description : undefined, + model: { + name: model.name, + fqn: model.fqn, + description: model.description, + }, + }), + ) + + const selected = await vscode.window.showQuickPick(items, { + placeHolder: 'Select a model for table diff', + }) + + if (!selected) { + return + } + + selectedModelInfo = selected.model + } else { + // Get the current document URI and check if it contains models + const documentUri = activeEditor.document.uri.toString(true) + + // Call the render model API to get models in the current file + const result = await lspClient.call_custom_method( + 'sqlmesh/render_model', + { + textDocumentUri: documentUri, + }, + ) + + if (isErr(result)) { + vscode.window.showErrorMessage( + `Failed to get models from current file: ${result.error.message}`, + ) + return + } + + // Check if we got any models + if (!result.value.models || result.value.models.length === 0) { + vscode.window.showInformationMessage( + 'No models found in the current file', + ) + return + } + + // If multiple models, let user choose + if (result.value.models.length > 1) { + const items = result.value.models.map(model => ({ + label: model.name, + description: model.fqn, + detail: model.description ? model.description : undefined, + model: model, + })) + + const selected = await vscode.window.showQuickPick(items, { + placeHolder: 'Select a model for table diff', + }) + + if (!selected) { + return + } + + selectedModelInfo = selected.model + } else { + selectedModelInfo = result.value.models[0] + } + } + + // Ensure we have a selected model + if (!selectedModelInfo) { + vscode.window.showErrorMessage('No model selected') + return + } + + // Get environments for selection + const environmentsResult = await lspClient.call_custom_method( + 'sqlmesh/get_environments', + {}, + ) + + if (isErr(environmentsResult)) { + vscode.window.showErrorMessage( + `Failed to get environments: ${environmentsResult.error.message}`, + ) + return + } + + const environments = environmentsResult.value.environments || {} + const environmentNames = Object.keys(environments) + + if (environmentNames.length === 0) { + vscode.window.showErrorMessage('No environments found') + return + } + + // Let user select source environment + const sourceEnvironmentItems = environmentNames.map(env => ({ + label: env, + description: `Source environment: ${env}`, + })) + + const selectedSourceEnv = await vscode.window.showQuickPick( + sourceEnvironmentItems, + { + placeHolder: 'Select source environment', + }, + ) + + if (!selectedSourceEnv) { + return + } + + // Let user select target environment (excluding source) + const targetEnvironmentItems = environmentNames + .filter(env => env !== selectedSourceEnv.label) + .map(env => ({ + label: env, + description: `Target environment: ${env}`, + })) + + if (targetEnvironmentItems.length === 0) { + vscode.window.showErrorMessage( + 'Need at least two environments for comparison', + ) + return + } + + const selectedTargetEnv = await vscode.window.showQuickPick( + targetEnvironmentItems, + { + placeHolder: 'Select target environment', + }, + ) + + if (!selectedTargetEnv) { + return + } + + // Run table diff immediately with selected parameters + const tableDiffResult = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'SQLMesh', + cancellable: false, + }, + async progress => { + progress.report({ message: 'Calculating table differences...' }) + + return await lspClient.call_custom_method('sqlmesh/api', { + method: 'GET', + endpoint: '/api/table_diff', + params: { + source: selectedSourceEnv.label, + target: selectedTargetEnv.label, + model_or_snapshot: selectedModelInfo.name, + }, + body: {}, + }) + }, + ) + + if (isErr(tableDiffResult)) { + vscode.window.showErrorMessage( + `Failed to run table diff: ${tableDiffResult.error.message}`, + ) + return + } + + // Determine the view column for side-by-side display + // Find the rightmost column with an editor + let maxColumn = vscode.ViewColumn.One + for (const editor of vscode.window.visibleTextEditors) { + if (editor.viewColumn && editor.viewColumn > maxColumn) { + maxColumn = editor.viewColumn + } + } + + // Open in the next column after the rightmost editor + const viewColumn = maxColumn + 1 + + // Create a webview panel for the table diff + const panel = vscode.window.createWebviewPanel( + 'sqlmesh.tableDiff', + `SQLMesh Table Diff - ${selectedModelInfo.name} (${selectedSourceEnv.label} → ${selectedTargetEnv.label})`, + viewColumn, + { + enableScripts: true, + retainContextWhenHidden: true, + localResourceRoots: [extensionUri], + }, + ) + + // Store the initial data for the webview + // eslint-disable-next-line prefer-const + let initialData = { + selectedModel: selectedModelInfo, + sourceEnvironment: selectedSourceEnv.label, + targetEnvironment: selectedTargetEnv.label, + tableDiffData: tableDiffResult.value, + environments: environments, + } + + // Set up message listener for events from the webview + panel.webview.onDidReceiveMessage( + async request => { + if (!request || !request.key) { + return + } + const message: CallbackEvent = request + switch (message.key) { + case 'openFile': { + const workspaceFolders = getWorkspaceFolders() + if (workspaceFolders.length != 1) { + throw new Error('Only one workspace folder is supported') + } + const fullPath = vscode.Uri.parse(message.payload.uri) + const document = await vscode.workspace.openTextDocument(fullPath) + await vscode.window.showTextDocument(document) + break + } + case 'rpcRequest': { + const payload: RPCRequest = message.payload + const requestId = payload.requestId + switch (payload.method) { + case 'api_query': { + const response = await lspClient.call_custom_method( + 'sqlmesh/api', + payload.params, + ) + let responseCallback: CallbackEvent + if (isErr(response)) { + let errorMessage: string + switch (response.error.type) { + case 'generic': + errorMessage = response.error.message + break + case 'invalid_state': + errorMessage = `Invalid state: ${response.error.message}` + break + case 'sqlmesh_outdated': + errorMessage = `SQLMesh version issue: ${response.error.message}` + break + default: + errorMessage = 'Unknown error' + } + responseCallback = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: false, + error: errorMessage, + }, + }, + } + } else { + responseCallback = { + key: 'rpcResponse', + payload: { + requestId, + result: response, + }, + } + } + await panel.webview.postMessage(responseCallback) + break + } + case 'get_active_file': { + const active_file = + vscode.window.activeTextEditor?.document.uri.fsPath + const responseCallback: CallbackEvent = { + key: 'rpcResponse', + payload: { + requestId, + result: { + fileUri: active_file, + }, + }, + } + await panel.webview.postMessage(responseCallback) + break + } + case 'get_selected_model': { + const responseCallback: CallbackEvent = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: true, + value: { + selectedModel: initialData.selectedModel, + }, + }, + }, + } + await panel.webview.postMessage(responseCallback) + break + } + case 'get_initial_data': { + const responseCallback: CallbackEvent = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: true, + value: { + selectedModel: initialData.selectedModel, + sourceEnvironment: initialData.sourceEnvironment, + targetEnvironment: initialData.targetEnvironment, + tableDiffData: initialData.tableDiffData, + environments: initialData.environments, + }, + }, + }, + } + await panel.webview.postMessage(responseCallback) + break + } + case 'get_all_models': { + const allModelsResult = await lspClient.call_custom_method( + 'sqlmesh/get_models', + {}, + ) + + let responseCallback: CallbackEvent + if (isErr(allModelsResult)) { + responseCallback = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: false, + error: `Failed to get models: ${allModelsResult.error.message}`, + }, + }, + } + } else { + responseCallback = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: true, + value: { + ok: true, + models: allModelsResult.value.models || [], + }, + }, + }, + } + } + await panel.webview.postMessage(responseCallback) + break + } + case 'set_selected_model': { + const modelInfo = payload.params?.model + if (modelInfo) { + initialData.selectedModel = modelInfo + // Update the panel title to reflect the new selection + panel.title = `SQLMesh Table Diff - ${modelInfo.name} (${initialData.sourceEnvironment} → ${initialData.targetEnvironment})` + } + + const responseCallback: CallbackEvent = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: true, + value: { + ok: true, + selectedModel: initialData.selectedModel, + }, + }, + }, + } + await panel.webview.postMessage(responseCallback) + break + } + case 'get_environments': { + const responseCallback: CallbackEvent = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: true, + value: { + ok: true, + environments: initialData.environments, + }, + }, + }, + } + await panel.webview.postMessage(responseCallback) + break + } + case 'run_table_diff': { + const { sourceModel, sourceEnvironment, targetEnvironment } = + payload.params || {} + + if (!sourceModel || !sourceEnvironment || !targetEnvironment) { + const responseCallback: CallbackEvent = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: false, + error: + 'Missing required parameters: sourceModel, sourceEnvironment, or targetEnvironment', + }, + }, + } + await panel.webview.postMessage(responseCallback) + break + } + + const tableDiffResult = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'SQLMesh', + cancellable: false, + }, + async progress => { + progress.report({ + message: 'Calculating table differences...', + }) + + return await lspClient.call_custom_method('sqlmesh/api', { + method: 'GET', + endpoint: '/api/table_diff', + params: { + source: sourceEnvironment, + target: targetEnvironment, + model_or_snapshot: sourceModel, + }, + body: {}, + }) + }, + ) + + let responseCallback: CallbackEvent + if (isErr(tableDiffResult)) { + responseCallback = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: false, + error: `Failed to run table diff: ${tableDiffResult.error.message}`, + }, + }, + } + } else { + responseCallback = { + key: 'rpcResponse', + payload: { + requestId, + result: { + ok: true, + value: { + ok: true, + data: tableDiffResult.value, + }, + }, + }, + } + } + await panel.webview.postMessage(responseCallback) + break + } + default: { + throw new Error(`Unhandled RPC method: ${payload.method}`) + } + } + break + } + default: + console.error( + 'Unhandled message type under queryRequest: ', + message, + ) + } + }, + undefined, + [], + ) + + // Set the HTML content + panel.webview.html = getHTML(panel.webview, extensionUri) + } +} + +function getHTML(webview: vscode.Webview, extensionUri: vscode.Uri): string { + const cssUri = webview.asWebviewUri( + vscode.Uri.joinPath(extensionUri, 'src_react', 'assets', 'index.css'), + ) + const jsUri = webview.asWebviewUri( + vscode.Uri.joinPath(extensionUri, 'src_react', 'assets', 'index.js'), + ) + const faviconUri = webview.asWebviewUri( + vscode.Uri.joinPath(extensionUri, 'src_react', 'favicon.ico'), + ) + const logoUri = webview.asWebviewUri( + vscode.Uri.joinPath(extensionUri, 'src_react', 'logo192.png'), + ) + + return ` + + + + + + + + + + SQLMesh Table Diff + + + + + +
+ + +` +} diff --git a/vscode/extension/src/extension.ts b/vscode/extension/src/extension.ts index 0d0f6252cb..cfea8c2228 100644 --- a/vscode/extension/src/extension.ts +++ b/vscode/extension/src/extension.ts @@ -30,6 +30,7 @@ import { handleError } from './utilities/errors' import { selector, completionProvider } from './completion/completion' import { LineagePanel } from './webviews/lineagePanel' import { RenderedModelProvider } from './providers/renderedModelProvider' +import { showTableDiff } from './commands/tableDiff' import { controller as testController, @@ -151,6 +152,14 @@ export async function activate(context: vscode.ExtensionContext) { ), ) + // Register the table diff command + context.subscriptions.push( + vscode.commands.registerCommand( + 'sqlmesh.showTableDiff', + showTableDiff(lspClient, context.extensionUri), + ), + ) + // Re‑render model automatically when its source file is saved context.subscriptions.push( vscode.workspace.onDidSaveTextDocument(async document => { diff --git a/vscode/extension/src/lsp/custom.ts b/vscode/extension/src/lsp/custom.ts index 8113cd86ae..152f316cdf 100644 --- a/vscode/extension/src/lsp/custom.ts +++ b/vscode/extension/src/lsp/custom.ts @@ -35,6 +35,8 @@ export type CustomLSPMethods = | ListWorkspaceTests | ListDocumentTests | RunTest + | GetEnvironmentsMethod + | GetTableDiffModelsMethod interface AllModelsRequest { textDocument: { @@ -176,3 +178,44 @@ export interface RunTestResponse extends BaseResponse { success: boolean error_message?: string } + +export interface GetEnvironmentsMethod { + method: 'sqlmesh/get_environments' + request: GetEnvironmentsRequest + response: GetEnvironmentsResponse +} + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +interface GetEnvironmentsRequest {} + +interface GetEnvironmentsResponse extends BaseResponse { + environments: Record + pinned_environments: string[] + default_target_environment: string +} + +interface EnvironmentInfo { + name: string + snapshots: string[] + start_at: string + plan_id: string +} + +export interface GetTableDiffModelsMethod { + method: 'sqlmesh/get_models' + request: GetModelsRequest + response: GetModelsResponse +} + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +interface GetModelsRequest {} + +interface GetModelsResponse extends BaseResponse { + models: ModelInfo[] +} + +interface ModelInfo { + name: string + fqn: string + description: string | null | undefined +} diff --git a/vscode/extension/src/webviews/lineagePanel.ts b/vscode/extension/src/webviews/lineagePanel.ts index ee05112a64..0fd0be9c2a 100644 --- a/vscode/extension/src/webviews/lineagePanel.ts +++ b/vscode/extension/src/webviews/lineagePanel.ts @@ -200,6 +200,9 @@ export class LineagePanel implements WebviewViewProvider, Disposable { /> Create TanStack App - react + diff --git a/vscode/react/src/api/instance.ts b/vscode/react/src/api/instance.ts index 3627b273de..781a98ef88 100644 --- a/vscode/react/src/api/instance.ts +++ b/vscode/react/src/api/instance.ts @@ -39,7 +39,7 @@ export async function fetchAPI( _options?: Partial, ): Promise { const request = { - url: config.url, + endpoint: config.url, method: config.method, params: config.params, body: config.data, diff --git a/vscode/react/src/components/tablediff/Card.tsx b/vscode/react/src/components/tablediff/Card.tsx new file mode 100644 index 0000000000..d2f4d833c2 --- /dev/null +++ b/vscode/react/src/components/tablediff/Card.tsx @@ -0,0 +1,51 @@ +import { type ReactNode } from 'react' +import { twColors, twMerge } from './tailwind-utils' + +interface CardProps { + children: ReactNode + className?: string +} + +export function Card({ children, className }: CardProps) { + return ( +
+ {children} +
+ ) +} + +interface CardHeaderProps { + children: ReactNode + className?: string +} + +export function CardHeader({ children, className }: CardHeaderProps) { + return ( +
+ {children} +
+ ) +} + +interface CardContentProps { + children: ReactNode + className?: string +} + +export function CardContent({ children, className }: CardContentProps) { + return
{children}
+} diff --git a/vscode/react/src/components/tablediff/ColumnStatsSection.tsx b/vscode/react/src/components/tablediff/ColumnStatsSection.tsx new file mode 100644 index 0000000000..6b65318864 --- /dev/null +++ b/vscode/react/src/components/tablediff/ColumnStatsSection.tsx @@ -0,0 +1,381 @@ +import { useState } from 'react' +import { type TableDiffData, type SampleValue } from './types' +import { twColors, twMerge } from './tailwind-utils' +import { Card } from './Card' +import { + ArrowsUpDownIcon, + ArrowsRightLeftIcon, +} from '@heroicons/react/24/outline' + +interface ColumnStatsSectionProps { + columnStats: TableDiffData['row_diff']['column_stats'] +} + +interface StatHeaderProps { + stat: string +} + +const StatHeader = ({ stat }: StatHeaderProps) => ( + + {stat} + +) + +interface StatCellProps { + value: SampleValue +} + +const StatCell = ({ value }: StatCellProps) => ( + + {typeof value === 'number' ? value.toFixed(1) : String(value)} + +) + +interface ColumnStatRowProps { + columnName: string + statsValue: TableDiffData['row_diff']['column_stats'][string] +} + +const ColumnStatRow = ({ columnName, statsValue }: ColumnStatRowProps) => ( + + + {columnName} + + {statsValue && typeof statsValue === 'object' + ? Object.values(statsValue as Record).map( + (value, idx) => ( + + ), + ) + : [ + , + ]} + +) + +export function ColumnStatsSection({ columnStats }: ColumnStatsSectionProps) { + const [isVertical, setIsVertical] = useState(false) + + if (Object.keys(columnStats || {}).length === 0) { + return null + } + + // Get the first stats object to determine the column headers + const firstStatsValue = Object.values(columnStats)[0] + const statKeys = + firstStatsValue && typeof firstStatsValue === 'object' + ? Object.keys(firstStatsValue as Record) + : [] + + return ( +
+ {/* Statistics Table Card */} + + {/* Toggle Button */} +
+ +
+ +
+ {isVertical ? ( + // Vertical layout: Each stat as a separate row + + + + + {Object.keys(columnStats).map(col => ( + + ))} + + + + {statKeys.map(stat => ( + + + {Object.entries(columnStats).map(([col, statsValue]) => ( + )[stat] + : statsValue + } + /> + ))} + + ))} + +
+ Column + + {col} +
+ {stat} +
+ ) : ( + // Horizontal layout: Original layout + + + + + {statKeys.map(stat => ( + + ))} + + + + {Object.entries(columnStats).map(([col, statsValue]) => ( + + ))} + +
+ Column +
+ )} +
+ + + {/* Summary Cards */} +
+ {(() => { + let percentages: { column: string; percentage: number }[] = [] + + if (columnStats && typeof columnStats === 'object') { + if ( + 'pct_match' in columnStats && + typeof columnStats.pct_match === 'object' && + columnStats.pct_match !== null + ) { + const pctMatchData = columnStats.pct_match as Record< + string, + number + > + percentages = Object.entries(pctMatchData) + .map(([col, value]) => ({ + column: col, + percentage: Number(value) || 0, + })) + .filter(item => !isNaN(item.percentage)) + } else { + percentages = Object.entries(columnStats) + .map(([col, stats]) => { + if (!stats || typeof stats !== 'object') return null + + const statsObj = stats as Record + const pctMatch = + statsObj.pct_match || + statsObj.match_pct || + statsObj.percentage || + 0 + + return { column: col, percentage: Number(pctMatch) } + }) + .filter( + (item): item is { column: string; percentage: number } => + item !== null && + !isNaN(item.percentage) && + item.column !== 'pct_match', + ) + } + } + + const validPercentages = percentages.map(p => p.percentage) + const highest = + percentages.length > 0 + ? percentages.find( + p => p.percentage === Math.max(...validPercentages), + ) + : null + const lowest = + percentages.length > 0 + ? percentages.find( + p => p.percentage === Math.min(...validPercentages), + ) + : null + const average = + validPercentages.length > 0 + ? validPercentages.reduce((a, b) => a + b, 0) / + validPercentages.length + : 0 + + return ( + <> + +
+
+
+ {highest ? `${highest.percentage.toFixed(1)}%` : 'N/A'} +
+
+ Highest Match +
+
+ {highest ? highest.column : 'No data'} +
+
+ + + +
+
+
+ {average > 0 ? `${average.toFixed(1)}%` : 'N/A'} +
+
+ Average Match +
+
+ Across {validPercentages.length} columns +
+
+ + + +
+
+
+ {lowest ? `${lowest.percentage.toFixed(1)}%` : 'N/A'} +
+
+ Lowest Match +
+
+ {lowest ? lowest.column : 'No data'} +
+
+ + + ) + })()} +
+
+ ) +} diff --git a/vscode/react/src/components/tablediff/ContentSections.tsx b/vscode/react/src/components/tablediff/ContentSections.tsx new file mode 100644 index 0000000000..dee8019e0c --- /dev/null +++ b/vscode/react/src/components/tablediff/ContentSections.tsx @@ -0,0 +1,78 @@ +import { SectionCard } from './SectionCard' +import { SchemaDiffSection } from './SchemaDiffSection' +import { RowStatsSection } from './RowStatsSection' +import { ColumnStatsSection } from './ColumnStatsSection' +import { SampleDataSection } from './SampleDataSection' +import { usePersistedState } from './hooks' +import type { TableDiffData, ExpandedSections } from './types' + +interface ContentSectionsProps { + data: TableDiffData +} + +export function ContentSections({ data }: ContentSectionsProps) { + const [expanded, setExpanded] = usePersistedState( + 'tableDiffExpanded', + { + schema: true, + rows: true, + columnStats: false, + sampleData: false, + }, + ) + + const toggle = (section: keyof ExpandedSections) => { + setExpanded(prev => ({ + ...prev, + [section]: !prev[section], + })) + } + + const { schema_diff, row_diff } = data + + return ( +
+ {/* Schema Changes */} + toggle('schema')} + > + + + + {/* Row Statistics */} + toggle('rows')} + > + + + + {/* Column Statistics */} + toggle('columnStats')} + > + + + + {/* Sample Data */} + {row_diff.processed_sample_data && ( + toggle('sampleData')} + > + + + )} +
+ ) +} diff --git a/vscode/react/src/components/tablediff/HeaderCard.tsx b/vscode/react/src/components/tablediff/HeaderCard.tsx new file mode 100644 index 0000000000..5c3b3872a6 --- /dev/null +++ b/vscode/react/src/components/tablediff/HeaderCard.tsx @@ -0,0 +1,201 @@ +import { Card, CardContent } from './Card' +import { DiffConfig } from './Legend' +import { twColors, twMerge } from './tailwind-utils' +import type { TableDiffData } from './types' + +interface HeaderCardProps { + schemaDiff: TableDiffData['schema_diff'] + rowDiff: TableDiffData['row_diff'] + limit: number + whereClause: string + onColumns: string + on: string[][] | undefined + where: string | undefined + isRerunning: boolean + onLimitChange: (limit: number) => void + onWhereClauseChange: (where: string) => void + onOnColumnsChange: (on: string) => void + onRerun: () => void + hasChanges: boolean +} + +export function HeaderCard({ + schemaDiff, + rowDiff, + limit, + whereClause, + onColumns, + on, + where, + isRerunning, + onLimitChange, + onWhereClauseChange, + onOnColumnsChange, + onRerun, + hasChanges, +}: HeaderCardProps) { + const formatPercentage = (v: number) => `${v.toFixed(1)}%` + const formatCount = (v: number) => v.toLocaleString() + + return ( + + +
+ + Source: + + + {schemaDiff.source} + + + Target: + + + {schemaDiff.target} + +
+
+
+ Source rows: + + {formatCount(rowDiff.source_count)} + +
+
+ Target rows: + + {formatCount(rowDiff.target_count)} + +
+
+ Change: + 0 + ? twColors.textSuccess500 + : rowDiff.count_pct_change < 0 + ? twColors.textDanger500 + : twColors.textMuted, + )} + > + {formatPercentage(rowDiff.count_pct_change)} + +
+
+
+
+
+ + + onLimitChange(Math.max(1, parseInt(e.target.value) || 1)) + } + className={twMerge( + 'w-20 px-2 py-1 text-sm rounded border', + 'bg-[var(--vscode-input-background)]', + 'border-[var(--vscode-input-border)]', + 'text-[var(--vscode-input-foreground)]', + 'focus:outline-none focus:ring-1 focus:ring-[var(--vscode-focusBorder)]', + )} + min="1" + max="10000" + disabled={isRerunning} + /> +
+
+ + onWhereClauseChange(e.target.value)} + placeholder="e.g. created_at > '2024-01-01'" + className={twMerge( + 'px-2 py-1 text-sm rounded border', + 'bg-[var(--vscode-input-background)]', + 'border-[var(--vscode-input-border)]', + 'text-[var(--vscode-input-foreground)]', + 'placeholder:text-[var(--vscode-input-placeholderForeground)]', + 'focus:outline-none focus:ring-1 focus:ring-[var(--vscode-focusBorder)]', + )} + disabled={isRerunning} + /> +
+
+ + onOnColumnsChange(e.target.value)} + placeholder="e.g. s.id = t.id AND s.date = t.date" + className={twMerge( + 'px-2 py-1 text-sm rounded border', + 'bg-[var(--vscode-input-background)]', + 'border-[var(--vscode-input-border)]', + 'text-[var(--vscode-input-foreground)]', + 'placeholder:text-[var(--vscode-input-placeholderForeground)]', + 'focus:outline-none focus:ring-1 focus:ring-[var(--vscode-focusBorder)]', + )} + disabled={isRerunning} + /> +
+ +
+
+ {on && ( + + )} +
+
+ + + ) +} diff --git a/vscode/react/src/components/tablediff/Legend.tsx b/vscode/react/src/components/tablediff/Legend.tsx new file mode 100644 index 0000000000..274db60625 --- /dev/null +++ b/vscode/react/src/components/tablediff/Legend.tsx @@ -0,0 +1,59 @@ +import { twColors, twMerge } from './tailwind-utils' + +interface DiffConfigProps { + on: string[] | string[][] + limit?: number + where?: string +} + +interface ConfigItemProps { + label: string + value: string | number +} + +function ConfigItem({ label, value }: ConfigItemProps) { + return ( +
+ + {label}: + + + {value} + +
+ ) +} + +export function DiffConfig({ on, limit, where }: DiffConfigProps) { + // Handle the grain (join keys) + const grainColumns = Array.isArray(on[0]) + ? on.flat().filter((col, index, arr) => arr.indexOf(col) === index) // Remove duplicates from nested array + : (on as string[]) + + return ( +
+ + {limit && ( + + )} + {where && ( + + )} +
+ ) +} diff --git a/vscode/react/src/components/tablediff/RerunController.tsx b/vscode/react/src/components/tablediff/RerunController.tsx new file mode 100644 index 0000000000..6a609ecaaa --- /dev/null +++ b/vscode/react/src/components/tablediff/RerunController.tsx @@ -0,0 +1,168 @@ +import { useState, useEffect } from 'react' +import { callRpc } from '../../utils/rpc' +import type { TableDiffData, TableDiffParams } from './types' + +interface RerunControllerProps { + data: TableDiffData + onDataUpdate?: (data: TableDiffData) => void + children: (props: { + limit: number + whereClause: string + onColumns: string + isRerunning: boolean + hasChanges: boolean + setLimit: (limit: number) => void + setWhereClause: (where: string) => void + setOnColumns: (on: string) => void + handleRerun: () => void + }) => React.ReactNode +} + +export function RerunController({ + data, + onDataUpdate, + children, +}: RerunControllerProps) { + const [isRerunning, setIsRerunning] = useState(false) + const [limit, setLimit] = useState(data.limit || 20) + const [whereClause, setWhereClause] = useState(data.where || '') + const [onColumns, setOnColumns] = useState( + data.on?.map(([sCol, tCol]) => `s.${sCol} = t.${tCol}`).join(' AND ') || '', + ) + + // Update state when data changes + useEffect(() => { + setLimit(data.limit || 20) + setWhereClause(data.where || '') + setOnColumns( + data.on?.map(([sCol, tCol]) => `s.${sCol} = t.${tCol}`).join(' AND ') || + '', + ) + }, [data.limit, data.where, data.on]) + + // Helper function to parse on columns back to array format + const parseOnColumns = (onString: string): string[][] => { + if (!onString.trim()) return [] + + // Parse "s.id = t.id AND s.date = t.date" back to [["id", "id"], ["date", "date"]] + const conditions = onString.split(' AND ') + return conditions.map(condition => { + const match = condition.trim().match(/^s\.(\w+)\s*=\s*t\.(\w+)$/) + if (match) { + return [match[1], match[2]] + } + // Fallback for simple format + return [condition.trim(), condition.trim()] + }) + } + + const hasChanges = + limit !== (data.limit || 20) || + whereClause !== (data.where || '') || + onColumns !== + (data.on?.map(([sCol, tCol]) => `s.${sCol} = t.${tCol}`).join(' AND ') || + '') + + const handleRerun = async () => { + if (isRerunning || !hasChanges) return + + setIsRerunning(true) + try { + // Get the initial data to extract the model name and environment names + const initialDataResult = await callRpc('get_initial_data', {}) + if (!initialDataResult.ok || !initialDataResult.value?.selectedModel) { + console.error('Failed to get initial data for rerun') + return + } + + const params: TableDiffParams = { + source: initialDataResult.value.sourceEnvironment || 'prod', + target: initialDataResult.value.targetEnvironment || 'dev', + model_or_snapshot: initialDataResult.value.selectedModel.name, + limit: Math.min(Math.max(1, limit), 10000), // Ensure limit is within bounds + ...(whereClause.trim() && { where: whereClause.trim() }), + ...(onColumns.trim() && { on: onColumns.trim() }), + } + + console.log('Rerunning table diff with params:', params) + + try { + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Request timeout')), 30000) // 30 second timeout + }) + + const apiPromise = callRpc('api_query', { + method: 'GET', + endpoint: '/api/table_diff', + params: params, + body: {}, + }) + + const result = (await Promise.race([apiPromise, timeoutPromise])) as any + + console.log('Table diff result:', result) + + if (result.ok && result.value) { + let newData: TableDiffData + if (result.value.data) { + newData = { + ...result.value.data, + limit, + where: whereClause, + on: parseOnColumns(onColumns), + } + } else { + newData = { + ...result.value, + limit, + where: whereClause, + on: parseOnColumns(onColumns), + } + } + + console.log('Updating table diff data:', newData) + onDataUpdate?.(newData) + } else { + console.error('API call failed:', result.error) + // Try to extract meaningful error message + let errorMessage = 'Unknown error' + if (typeof result.error === 'string') { + try { + const parsed = JSON.parse(result.error) + errorMessage = parsed.message || parsed.code || result.error + } catch { + errorMessage = result.error + } + } + console.error('Processed error message:', errorMessage) + setIsRerunning(false) + return + } + } catch (apiError) { + console.error('API call threw exception:', apiError) + setIsRerunning(false) + return + } + } catch (error) { + console.error('Error rerunning table diff:', error) + } finally { + setIsRerunning(false) + } + } + + return ( + <> + {children({ + limit, + whereClause, + onColumns, + isRerunning, + hasChanges, + setLimit, + setWhereClause, + setOnColumns, + handleRerun, + })} + + ) +} diff --git a/vscode/react/src/components/tablediff/RowStatsSection.tsx b/vscode/react/src/components/tablediff/RowStatsSection.tsx new file mode 100644 index 0000000000..076f7d95fe --- /dev/null +++ b/vscode/react/src/components/tablediff/RowStatsSection.tsx @@ -0,0 +1,102 @@ +import { type TableDiffData } from './types' +import { twColors, twMerge } from './tailwind-utils' +import { Card, CardContent } from './Card' + +interface RowStatsSectionProps { + rowDiff: TableDiffData['row_diff'] +} + +export function RowStatsSection({ rowDiff }: RowStatsSectionProps) { + const formatPercentage = (v: number) => `${(v * 100).toFixed(1)}%` + const formatCount = (v: number) => v.toLocaleString() + + const fullMatchCount = Math.round(rowDiff.stats.full_match_count || 0) + const joinCount = Math.round(rowDiff.stats.join_count || 0) + const partialMatchCount = joinCount - fullMatchCount + const sOnlyCount = Math.round(rowDiff.stats.s_only_count || 0) + const tOnlyCount = Math.round(rowDiff.stats.t_only_count || 0) + const totalRows = rowDiff.source_count + rowDiff.target_count + const fullMatchPct = totalRows > 0 ? (2 * fullMatchCount) / totalRows : 0 + + return ( +
+ {/* Full Match Card */} + +
+ +
+ {formatCount(fullMatchCount)} +
+
+ Full Matches +
+
+ {formatPercentage(fullMatchPct)} +
+ + + + {/* Partial Match Card */} + +
+ +
+ {formatCount(partialMatchCount)} +
+
+ Partial Matches +
+
+ {formatPercentage(partialMatchCount / totalRows)} +
+ + + + {/* Source Only Card */} + +
+ +
+ {formatCount(sOnlyCount)} +
+
+ Source Only +
+
+ {formatPercentage(sOnlyCount / totalRows)} +
+ + + + {/* Target Only Card */} + +
+ +
+ {formatCount(tOnlyCount)} +
+
+ Target Only +
+
+ {formatPercentage(tOnlyCount / totalRows)} +
+ + +
+ ) +} diff --git a/vscode/react/src/components/tablediff/SampleDataSection.tsx b/vscode/react/src/components/tablediff/SampleDataSection.tsx new file mode 100644 index 0000000000..00a7f07269 --- /dev/null +++ b/vscode/react/src/components/tablediff/SampleDataSection.tsx @@ -0,0 +1,440 @@ +import { useMemo } from 'react' +import { + type TableDiffData, + type SampleRow, + type SampleValue, + formatCellValue, +} from './types' +import { twColors, twMerge } from './tailwind-utils' + +interface SampleDataSectionProps { + rowDiff: TableDiffData['row_diff'] +} + +interface TableHeaderCellProps { + columnKey: string + sourceName?: SampleValue + targetName?: SampleValue +} + +const TableHeaderCell = ({ + columnKey, + sourceName, + targetName, +}: TableHeaderCellProps) => { + const isSource = columnKey === sourceName + const isTarget = columnKey === targetName + + return ( + + {columnKey} + + ) +} + +interface DiffTableCellProps { + columnKey: string + value: SampleValue + sourceName?: SampleValue + targetName?: SampleValue + decimals?: number +} + +const DiffTableCell = ({ + columnKey, + value, + sourceName, + targetName, + decimals = 3, +}: DiffTableCellProps) => { + const isSource = columnKey === sourceName + const isTarget = columnKey === targetName + + return ( + + {formatCellValue(value, decimals)} + + ) +} + +interface DiffTableRowProps { + row: SampleRow + sourceName?: SampleValue + targetName?: SampleValue + decimals?: number +} + +const DiffTableRow = ({ + row, + sourceName, + targetName, + decimals, +}: DiffTableRowProps) => ( + + {Object.entries(row) + .filter(([key]) => !key.startsWith('__')) + .map(([key, cell]) => ( + + ))} + +) + +interface SimpleTableCellProps { + value: SampleValue + colorClass: string + decimals?: number +} + +const SimpleTableCell = ({ + value, + colorClass, + decimals = 3, +}: SimpleTableCellProps) => ( + + {formatCellValue(value, decimals)} + +) + +interface SimpleTableRowProps { + row: SampleRow + colorClass: string + borderColorClass: string + decimals?: number +} + +const SimpleTableRow = ({ + row, + colorClass, + borderColorClass, + decimals, +}: SimpleTableRowProps) => ( + + {Object.values(row).map((cell, cellIdx) => ( + + ))} + +) + +interface ColumnDifferenceGroupProps { + columnName: string + rows: SampleRow[] + decimals: number +} + +const ColumnDifferenceGroup = ({ + columnName, + rows, + decimals, +}: ColumnDifferenceGroupProps) => { + if (!rows || rows.length === 0) return null + + const sourceName = rows[0].__source_name__ + const targetName = rows[0].__target_name__ + + return ( +
+
+ Column: {columnName} + + {rows.length} difference{rows.length > 1 ? 's' : ''} + +
+
+
+ + + + {Object.keys(rows[0] || {}) + .filter(key => !key.startsWith('__')) + .map(key => ( + + ))} + + + + {rows.slice(0, 10).map((row, rowIdx) => ( + + ))} + +
+
+ {rows.length > 10 && ( +

+ Showing first 10 of {rows.length} differing rows +

+ )} +
+
+ ) +} + +export function SampleDataSection({ rowDiff }: SampleDataSectionProps) { + const { processed_sample_data, decimals = 3 } = rowDiff + + if (!processed_sample_data) { + return ( +
+

+ No processed sample data available +

+
+ ) + } + + const { column_differences, source_only, target_only } = processed_sample_data + + // Group column differences by column name + const groupedDifferences = useMemo(() => { + const groups: Record = {} + + column_differences.forEach((row: SampleRow) => { + const columnName = String(row.__column_name__ || 'unknown') + if (!groups[columnName]) { + groups[columnName] = [] + } + groups[columnName].push(row) + }) + + return groups + }, [column_differences]) + + return ( +
+ {/* COMMON ROWS diff */} +
+

+ Common Rows +

+ {Object.keys(groupedDifferences).length > 0 ? ( +
+ {Object.entries(groupedDifferences).map(([columnName, rows]) => ( + + ))} +
+ ) : ( +

+ ✓ All joined rows match +

+ )} +
+ + {/* SOURCE ONLY & TARGET ONLY tables */} + {source_only && source_only.length > 0 && ( +
+

+ Source Only Rows +

+
+
+ + + + {Object.keys(source_only[0] || {}).map(col => ( + + ))} + + + + {source_only.slice(0, 10).map((row, rowIdx) => ( + + ))} + +
+ {col} +
+
+ {source_only.length > 10 && ( +
+ Showing first 10 of {source_only.length} rows +
+ )} +
+
+ )} + + {target_only && target_only.length > 0 && ( +
+

+ Target Only Rows +

+
+
+ + + + {Object.keys(target_only[0] || {}).map(col => ( + + ))} + + + + {target_only.slice(0, 10).map((row, rowIdx) => ( + + ))} + +
+ {col} +
+
+ {target_only.length > 10 && ( +
+ Showing first 10 of {target_only.length} rows +
+ )} +
+
+ )} +
+ ) +} diff --git a/vscode/react/src/components/tablediff/SchemaDiffSection.tsx b/vscode/react/src/components/tablediff/SchemaDiffSection.tsx new file mode 100644 index 0000000000..274ac1979c --- /dev/null +++ b/vscode/react/src/components/tablediff/SchemaDiffSection.tsx @@ -0,0 +1,122 @@ +import { useMemo } from 'react' +import { type TableDiffData } from './types' +import { twColors, twMerge } from './tailwind-utils' + +interface SchemaDiffSectionProps { + schemaDiff: TableDiffData['schema_diff'] +} + +interface SchemaChangeItemProps { + column: string + type: string + changeType: 'added' | 'removed' | 'modified' +} + +const SchemaChangeItem = ({ + column, + type, + changeType, +}: SchemaChangeItemProps) => { + const styleMap = { + added: { + bgClass: twColors.bgSuccess10, + borderClass: 'border-l-4 ' + twColors.borderSuccess500, + textClass: twColors.textSuccess500, + symbol: '+', + }, + removed: { + bgClass: twColors.bgDanger10, + borderClass: 'border-l-4 ' + twColors.borderDanger500, + textClass: twColors.textDanger500, + symbol: '-', + }, + modified: { + bgClass: twColors.bgPrimary10, + borderClass: 'border-l-4 ' + twColors.borderPrimary, + textClass: twColors.textPrimary, + symbol: '~', + }, + } + + const { bgClass, borderClass, textClass, symbol } = styleMap[changeType] + + return ( +
+ + {symbol} + + + {column} + + : + + {type} + +
+ ) +} + +export function SchemaDiffSection({ schemaDiff }: SchemaDiffSectionProps) { + const schemaHasChanges = useMemo(() => { + return ( + Object.keys(schemaDiff.added || {}).length > 0 || + Object.keys(schemaDiff.removed || {}).length > 0 || + Object.keys(schemaDiff.modified || {}).length > 0 + ) + }, [schemaDiff]) + + return ( +
+ {!schemaHasChanges ? ( +
+ ✓ Schemas are identical +
+ ) : ( + <> + {Object.entries(schemaDiff.added).map(([col, type]) => ( + + ))} + {Object.entries(schemaDiff.removed).map(([col, type]) => ( + + ))} + {Object.entries(schemaDiff.modified).map(([col, type]) => ( + + ))} + + )} +
+ ) +} diff --git a/vscode/react/src/components/tablediff/SectionCard.tsx b/vscode/react/src/components/tablediff/SectionCard.tsx new file mode 100644 index 0000000000..af9d42ecd2 --- /dev/null +++ b/vscode/react/src/components/tablediff/SectionCard.tsx @@ -0,0 +1,63 @@ +import { type ReactNode } from 'react' +import { ChevronDownIcon, ChevronRightIcon } from '@heroicons/react/24/outline' +import { Card, CardHeader, CardContent } from './Card' +import { twColors, twMerge } from './tailwind-utils' + +interface Props { + id: string + title: string + children: ReactNode + expanded: boolean + onToggle: () => void + badge?: { text: string; color?: string } +} + +export function SectionCard({ + title, + children, + expanded, + onToggle, + badge, +}: Props) { + return ( + + + + +
+ {children} +
+ + ) +} diff --git a/vscode/react/src/components/tablediff/SectionToggle.tsx b/vscode/react/src/components/tablediff/SectionToggle.tsx new file mode 100644 index 0000000000..4066db22b7 --- /dev/null +++ b/vscode/react/src/components/tablediff/SectionToggle.tsx @@ -0,0 +1,47 @@ +import { type ReactNode } from 'react' +import { ChevronDownIcon, ChevronRightIcon } from '@heroicons/react/24/outline' +import { type ExpandedSections } from './types' +import { twColors, twMerge } from './tailwind-utils' + +interface SectionToggleProps { + id: keyof ExpandedSections + title: string + expanded: boolean + onToggle(): void + children: ReactNode +} + +export function SectionToggle({ + title, + expanded, + onToggle, + children, +}: SectionToggleProps) { + return ( +
+ +
+ {children} +
+
+ ) +} diff --git a/vscode/react/src/components/tablediff/TableDiff.tsx b/vscode/react/src/components/tablediff/TableDiff.tsx new file mode 100644 index 0000000000..8374f1bc73 --- /dev/null +++ b/vscode/react/src/components/tablediff/TableDiff.tsx @@ -0,0 +1,148 @@ +import { useState, useEffect } from 'react' +import LoadingStatus from '../loading/LoadingStatus' +import { TableDiffResults } from './TableDiffResults' +import { callRpc } from '../../utils/rpc' +import { type TableDiffData } from './types' + +interface ModelInfo { + name: string + fqn: string + description?: string +} + +export function TableDiff() { + const [selectedModel, setSelectedModel] = useState(null) + const [sourceEnvironment, setSourceEnvironment] = useState('prod') + const [targetEnvironment, setTargetEnvironment] = useState('dev') + const [tableDiffData, setTableDiffData] = useState(null) + const [isLoadingDiff] = useState(false) + const [diffError] = useState(null) + const [hasInitialData, setHasInitialData] = useState(false) + + const handleDataUpdate = (newData: TableDiffData) => { + setTableDiffData(newData) + } + + // Load initial data on mount + useEffect(() => { + const loadInitialData = async () => { + try { + // Try to get initial data first (pre-selected from VSCode) + const initialDataResult = await callRpc('get_initial_data', {}) + if (initialDataResult.ok && initialDataResult.value) { + const data = initialDataResult.value + + // Set all initial state from pre-selected data + if (data.selectedModel) { + setSelectedModel(data.selectedModel) + } + if (data.sourceEnvironment) { + setSourceEnvironment(data.sourceEnvironment) + } + if (data.targetEnvironment) { + setTargetEnvironment(data.targetEnvironment) + } + + // Always mark as having initial data if we got a response from VSCode + setHasInitialData(true) + + if (data.tableDiffData) { + // Handle different response structures + let diffData: TableDiffData | null = null + + if (data.tableDiffData.data !== undefined) { + // Response has a nested data field + diffData = data.tableDiffData.data + } else if ( + data.tableDiffData && + typeof data.tableDiffData === 'object' && + 'schema_diff' in data.tableDiffData && + 'row_diff' in data.tableDiffData + ) { + // Response is the data directly + diffData = data.tableDiffData as TableDiffData + } + + setTableDiffData(diffData) + } + } + } catch (error) { + console.error('Error loading initial data:', error) + } + } + + loadInitialData() + }, []) + + // If we're still loading, show loading state + if (isLoadingDiff) { + return ( +
+ Running table diff... +
+ ) + } + + // If we have initial data, handle all possible states + if (hasInitialData) { + // Show results if we have them + if (tableDiffData) { + return ( +
+ +
+ ) + } + + // Show error if there was one + if (diffError) { + return ( +
+
+
+ Error running table diff +
+
{diffError}
+
+
+ ) + } + + // If we have initial data but no results and no error, show appropriate message + return ( +
+
+
No differences found
+
+ The selected model "{selectedModel?.name}" has no differences + between {sourceEnvironment}{' '} + and {targetEnvironment}{' '} + environments. +
+
+
+ ) + } + + // If we don't have initial data yet, show loading + if (!hasInitialData) { + return ( +
+ Loading... +
+ ) + } + + // This should never happen with the new flow + return ( +
+
+
Unexpected state
+
Please try running the table diff command again.
+
+
+ ) +} diff --git a/vscode/react/src/components/tablediff/TableDiffResults.tsx b/vscode/react/src/components/tablediff/TableDiffResults.tsx new file mode 100644 index 0000000000..45e92b65d1 --- /dev/null +++ b/vscode/react/src/components/tablediff/TableDiffResults.tsx @@ -0,0 +1,64 @@ +import { HeaderCard } from './HeaderCard' +import { ContentSections } from './ContentSections' +import { RerunController } from './RerunController' +import { type TableDiffData } from './types' +import { twColors, twMerge } from './tailwind-utils' + +interface Props { + data: TableDiffData + onDataUpdate?: (data: TableDiffData) => void +} + +export function TableDiffResults({ data, onDataUpdate }: Props) { + if (!data) + return ( +
+ No data available +
+ ) + + return ( + + {({ + limit, + whereClause, + onColumns, + isRerunning, + hasChanges, + setLimit, + setWhereClause, + setOnColumns, + handleRerun, + }) => ( +
+ + +
+ )} + + ) +} diff --git a/vscode/react/src/components/tablediff/hooks.ts b/vscode/react/src/components/tablediff/hooks.ts new file mode 100644 index 0000000000..803b0b8a16 --- /dev/null +++ b/vscode/react/src/components/tablediff/hooks.ts @@ -0,0 +1,29 @@ +import { useState, useEffect } from 'react' + +/** + * Persist state in localStorage so the user's expand / collapse choices + * survive reloads and navigation in VS Code's WebView. + */ +export function usePersistedState( + key: string, + initial: T, +): [T, React.Dispatch>] { + const [state, setState] = useState(() => { + try { + const stored = localStorage.getItem(key) + return stored ? (JSON.parse(stored) as T) : initial + } catch { + return initial + } + }) + + useEffect(() => { + try { + localStorage.setItem(key, JSON.stringify(state)) + } catch { + /* noop */ + } + }, [key, state]) + + return [state, setState] +} diff --git a/vscode/react/src/components/tablediff/index.ts b/vscode/react/src/components/tablediff/index.ts new file mode 100644 index 0000000000..a5b7ea2776 --- /dev/null +++ b/vscode/react/src/components/tablediff/index.ts @@ -0,0 +1,15 @@ +// Main components +export { TableDiff } from './TableDiff' +export { TableDiffResults } from './TableDiffResults' + +// Section components +export { SectionToggle } from './SectionToggle' +export { SchemaDiffSection } from './SchemaDiffSection' +export { RowStatsSection } from './RowStatsSection' +export { ColumnStatsSection } from './ColumnStatsSection' +export { SampleDataSection } from './SampleDataSection' + +// Utilities +export { usePersistedState } from './hooks' +export { twColors, twMerge } from './tailwind-utils' +export * from './types' diff --git a/vscode/react/src/components/tablediff/tailwind-utils.ts b/vscode/react/src/components/tablediff/tailwind-utils.ts new file mode 100644 index 0000000000..182dc69c28 --- /dev/null +++ b/vscode/react/src/components/tablediff/tailwind-utils.ts @@ -0,0 +1,86 @@ +// Tailwind utility classes with CSS variables +export const twColors = { + // Text colors + textForeground: 'text-[var(--vscode-editor-foreground)]', + textInfo: 'text-[var(--vscode-testing-iconUnset)]', + textSuccess: 'text-[var(--vscode-testing-iconPassed)]', + textError: 'text-[var(--vscode-testing-iconFailed)]', + textWarning: 'text-[var(--vscode-testing-iconQueued)]', + textMuted: 'text-[var(--vscode-descriptionForeground)]', + textAccent: 'text-[var(--vscode-textLink-foreground)]', + textAdded: 'text-[var(--vscode-diffEditor-insertedTextForeground)]', + textRemoved: 'text-[var(--vscode-diffEditor-removedTextForeground)]', + textModified: 'text-[var(--vscode-diffEditor-modifiedTextForeground)]', + + // Source and target environment colors + textSource: 'text-[var(--vscode-debugIcon-continueForeground)]', + textTarget: 'text-[var(--vscode-debugIcon-startForeground)]', + textClass: 'text-[var(--vscode-symbolIcon-classForeground)]', + bgSource: 'bg-[var(--vscode-debugIcon-continueForeground)]', + bgTarget: 'bg-[var(--vscode-debugIcon-startForeground)]', + bgClass: 'bg-[var(--vscode-symbolIcon-classForeground)]', + borderSource: 'border-[var(--vscode-debugIcon-continueForeground)]', + borderTarget: 'border-[var(--vscode-debugIcon-startForeground)]', + borderClass: 'border-[var(--vscode-symbolIcon-classForeground)]', + + // Background colors + bgEditor: 'bg-[var(--vscode-editor-background)]', + bgInput: 'bg-[var(--vscode-input-background)]', + bgHover: 'hover:bg-[var(--vscode-list-hoverBackground)]', + bgInactiveSelection: 'bg-[var(--vscode-editor-inactiveSelectionBackground)]', + bgAdded: 'bg-[var(--vscode-diffEditor-insertedTextBackground)]', + bgRemoved: 'bg-[var(--vscode-diffEditor-removedTextBackground)]', + bgModified: 'bg-[var(--vscode-diffEditor-modifiedTextBackground)]', + bgTestSuccess: 'bg-[var(--vscode-testing-iconPassed)]', + bgError: 'bg-[var(--vscode-testing-iconFailed)]', + bgWarning: 'bg-[var(--vscode-testing-iconQueued)]', + bgInfo: 'bg-[var(--vscode-testing-iconUnset)]', + + // Border colors + borderPanel: 'border-[var(--vscode-panel-border)]', + borderInfo: 'border-[var(--vscode-testing-iconUnset)]', + borderSuccess: 'border-[var(--vscode-testing-iconPassed)]', + borderError: 'border-[var(--vscode-diffEditor-removedTextForeground)]', + borderWarning: 'border-[var(--vscode-diffEditor-modifiedTextForeground)]', + borderAdded: 'border-[var(--vscode-diffEditor-insertedTextForeground)]', + borderRemoved: 'border-[var(--vscode-diffEditor-removedTextForeground)]', + borderModified: 'border-[var(--vscode-diffEditor-modifiedTextForeground)]', + + //These colors are similar to web UI + // Primary (blue) + textPrimary: 'text-[#3b82f6]', + bgPrimary10: 'bg-[#3b82f6]/10', + bgPrimary: 'bg-[#3b82f6]', + borderPrimary: 'border-[#3b82f6]', + + // Success (green) + textSuccess500: 'text-[#10b981]', + bgSuccess10: 'bg-[#10b981]/10', + bgSuccess: 'bg-[#10b981]', + borderSuccess500: 'border-[#10b981]', + + // Danger (red) + textDanger500: 'text-[#ef4444]', + bgDanger5: 'bg-[#ef4444]/5', + bgDanger10: 'bg-[#ef4444]/10', + bgDanger: 'bg-[#ef4444]', + borderDanger500: 'border-[#ef4444]', + + // Brand (purple) + textBrand: 'text-[#8b5cf6]', + bgBrand10: 'bg-[#8b5cf6]/10', + bgBrand: 'bg-[#8b5cf6]', + borderBrand500: 'border-[#8b5cf6]', + + // Neutral + bgNeutral5: 'bg-[var(--vscode-editor-inactiveSelectionBackground)]', + bgNeutral10: 'bg-[var(--vscode-list-hoverBackground)]', + textNeutral500: 'text-[var(--vscode-descriptionForeground)]', + textNeutral600: 'text-[var(--vscode-editor-foreground)]', + borderNeutral100: 'border-[var(--vscode-panel-border)]', +} + +// Helper function to combine conditional classes +export function twMerge(...classes: (string | false | undefined | null)[]) { + return classes.filter(Boolean).join(' ') +} diff --git a/vscode/react/src/components/tablediff/types.ts b/vscode/react/src/components/tablediff/types.ts new file mode 100644 index 0000000000..271828476b --- /dev/null +++ b/vscode/react/src/components/tablediff/types.ts @@ -0,0 +1,84 @@ +// Type for data values in samples - can be strings, numbers, booleans, or null +export type SampleValue = string | number | boolean | null + +// Type for row data in samples +export type SampleRow = Record + +// Type for column statistics +export type ColumnStats = Record + +export interface TableDiffData { + schema_diff: { + source: string + target: string + source_schema: Record + target_schema: Record + added: Record + removed: Record + modified: Record + } + row_diff: { + source: string + target: string + stats: Record + sample: Record + joined_sample: Record + s_sample: Record + t_sample: Record + column_stats: ColumnStats + source_count: number + target_count: number + count_pct_change: number + decimals: number + processed_sample_data?: { + column_differences: SampleRow[] + source_only: SampleRow[] + target_only: SampleRow[] + } + } + on: string[][] + limit?: number + where?: string +} + +export interface TableDiffParams { + source: string + target: string + model_or_snapshot: string + on?: string + where?: string + temp_schema?: string + limit?: number +} + +export interface ExpandedSections { + schema: boolean + rows: boolean + columnStats: boolean + sampleData: boolean +} + +export const themeColors = { + success: 'var(--vscode-testing-iconPassed, #22c55e)', + warning: 'var(--vscode-testing-iconQueued, #f59e0b)', + error: 'var(--vscode-testing-iconFailed, #ef4444)', + info: 'var(--vscode-testing-iconUnset, #3b82f6)', + addedText: 'var(--vscode-diffEditor-insertedTextForeground, #22c55e)', + removedText: 'var(--vscode-diffEditor-removedTextForeground, #ef4444)', + modifiedText: 'var(--vscode-diffEditor-modifiedTextForeground, #f59e0b)', + muted: 'var(--vscode-descriptionForeground)', + accent: 'var(--vscode-textLink-foreground)', + border: 'var(--vscode-panel-border)', +} + +// Helper utilities +export function cn(...classes: (string | false | undefined)[]) { + return classes.filter(Boolean).join(' ') +} + +export const formatCellValue = (cell: SampleValue, decimals = 3): string => { + if (cell == null) return 'null' + if (typeof cell === 'number') + return cell % 1 === 0 ? cell.toString() : cell.toFixed(decimals) + return String(cell) +} diff --git a/vscode/react/src/main.tsx b/vscode/react/src/main.tsx index cf3b691223..5e24fc648f 100644 --- a/vscode/react/src/main.tsx +++ b/vscode/react/src/main.tsx @@ -1,38 +1,37 @@ import { StrictMode } from 'react' import ReactDOM from 'react-dom/client' -import { RouterProvider, createRouter } from '@tanstack/react-router' - -// Import the generated route tree -import { routeTree } from './routeTree.gen' - import reportWebVitals from './reportWebVitals.ts' import { EventBusProvider } from './hooks/eventBus.tsx' +import { TableDiffPage } from './pages/tablediff.tsx' +import { LineagePage } from './pages/lineage.tsx' -// Create a new router instance -const router = createRouter({ - routeTree, - context: {}, - defaultPreload: 'intent', - scrollRestoration: true, - defaultStructuralSharing: true, - defaultPreloadStaleTime: 0, -}) +// Detect panel type +declare global { + interface Window { + __SQLMESH_PANEL_TYPE__?: string + } +} -// Register the router instance for type safety -declare module '@tanstack/react-router' { - interface Register { - router: typeof router +const panelType = window.__SQLMESH_PANEL_TYPE__ || 'lineage' + +// component selector +function App() { + if (panelType === 'tablediff') { + return } + + return } // Render the app const rootElement = document.getElementById('app') if (rootElement && !rootElement.innerHTML) { const root = ReactDOM.createRoot(rootElement) + root.render( - + , ) diff --git a/vscode/react/src/pages/tablediff.tsx b/vscode/react/src/pages/tablediff.tsx new file mode 100644 index 0000000000..47e3b4ed58 --- /dev/null +++ b/vscode/react/src/pages/tablediff.tsx @@ -0,0 +1,22 @@ +import '../App.css' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { TableDiff } from '../components/tablediff/TableDiff' + +export function TableDiffPage() { + const client = new QueryClient({ + defaultOptions: { + queries: { + networkMode: 'always', + refetchOnWindowFocus: false, + retry: false, + staleTime: Infinity, + }, + }, + }) + + return ( + + + + ) +} diff --git a/vscode/react/src/routeTree.gen.ts b/vscode/react/src/routeTree.gen.ts index dd198661f1..f18a46802a 100644 --- a/vscode/react/src/routeTree.gen.ts +++ b/vscode/react/src/routeTree.gen.ts @@ -9,9 +9,15 @@ // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. import { Route as rootRouteImport } from './routes/__root' +import { Route as TablediffRouteImport } from './routes/tablediff' import { Route as LineageRouteImport } from './routes/lineage' import { Route as IndexRouteImport } from './routes/index' +const TablediffRoute = TablediffRouteImport.update({ + id: '/tablediff', + path: '/tablediff', + getParentRoute: () => rootRouteImport, +} as any) const LineageRoute = LineageRouteImport.update({ id: '/lineage', path: '/lineage', @@ -26,31 +32,42 @@ const IndexRoute = IndexRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof IndexRoute '/lineage': typeof LineageRoute + '/tablediff': typeof TablediffRoute } export interface FileRoutesByTo { '/': typeof IndexRoute '/lineage': typeof LineageRoute + '/tablediff': typeof TablediffRoute } export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute '/lineage': typeof LineageRoute + '/tablediff': typeof TablediffRoute } export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath - fullPaths: '/' | '/lineage' + fullPaths: '/' | '/lineage' | '/tablediff' fileRoutesByTo: FileRoutesByTo - to: '/' | '/lineage' - id: '__root__' | '/' | '/lineage' + to: '/' | '/lineage' | '/tablediff' + id: '__root__' | '/' | '/lineage' | '/tablediff' fileRoutesById: FileRoutesById } export interface RootRouteChildren { IndexRoute: typeof IndexRoute LineageRoute: typeof LineageRoute + TablediffRoute: typeof TablediffRoute } declare module '@tanstack/react-router' { interface FileRoutesByPath { + '/tablediff': { + id: '/tablediff' + path: '/tablediff' + fullPath: '/tablediff' + preLoaderRoute: typeof TablediffRouteImport + parentRoute: typeof rootRouteImport + } '/lineage': { id: '/lineage' path: '/lineage' @@ -71,6 +88,7 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, LineageRoute: LineageRoute, + TablediffRoute: TablediffRoute, } export const routeTree = rootRouteImport ._addFileChildren(rootRouteChildren) diff --git a/vscode/react/src/routes/__root.tsx b/vscode/react/src/routes/__root.tsx index 83a1d6db65..a600d0f849 100644 --- a/vscode/react/src/routes/__root.tsx +++ b/vscode/react/src/routes/__root.tsx @@ -2,6 +2,7 @@ import { Outlet, createRootRoute } from '@tanstack/react-router' import { TanStackRouterDevtools } from '@tanstack/react-router-devtools' import '../App.css' import { LineagePage } from '@/pages/lineage' +import { TableDiffPage } from '@/pages/tablediff' export const Route = createRootRoute({ component: () => { @@ -13,6 +14,8 @@ export const Route = createRootRoute({ ) }, notFoundComponent: () => { - return + // switch to lineage or table diff based on panel type + const panelType = (window as any).__SQLMESH_PANEL_TYPE__ || 'lineage' + return panelType === 'tablediff' ? : }, }) diff --git a/vscode/react/src/routes/tablediff.tsx b/vscode/react/src/routes/tablediff.tsx new file mode 100644 index 0000000000..c9776048cd --- /dev/null +++ b/vscode/react/src/routes/tablediff.tsx @@ -0,0 +1,6 @@ +import { createFileRoute } from '@tanstack/react-router' +import { TableDiffPage } from '../pages/tablediff' + +export const Route = createFileRoute('/tablediff')({ + component: TableDiffPage, +}) diff --git a/web/server/api/endpoints/table_diff.py b/web/server/api/endpoints/table_diff.py index 6d11638b84..d441d49e5a 100644 --- a/web/server/api/endpoints/table_diff.py +++ b/web/server/api/endpoints/table_diff.py @@ -6,12 +6,109 @@ from sqlglot import exp from sqlmesh.core.context import Context -from web.server.models import RowDiff, SchemaDiff, TableDiff +from web.server.models import ProcessedSampleData, RowDiff, SchemaDiff, TableDiff from web.server.settings import get_loaded_context router = APIRouter() +def _cells_match(x: t.Any, y: t.Any) -> bool: + # lazily import pandas and numpy as we do in core + import pandas as pd + import numpy as np + + def _normalize(val: t.Any) -> t.Any: + if pd.isnull(val): + val = None + return list(val) if isinstance(val, (pd.Series, np.ndarray)) else val + + return _normalize(x) == _normalize(y) + + +def _process_sample_data( + row_diff: t.Any, source_name: str, target_name: str +) -> ProcessedSampleData: + import pandas as pd + + if row_diff.joined_sample.shape[0] == 0: + return ProcessedSampleData( + column_differences=[], + source_only=row_diff.s_sample.replace({pd.NA: None}).to_dict("records") + if row_diff.s_sample.shape[0] > 0 + else [], + target_only=row_diff.t_sample.replace({pd.NA: None}).to_dict("records") + if row_diff.t_sample.shape[0] > 0 + else [], + ) + + keys: list[str] = [] + columns: dict[str, list[str]] = {} + + # todo: to be refactored to the diff module itself since it is similar to console + source_prefix, source_display = ( + (f"{source_name}__", source_name.upper()) + if source_name.lower() != row_diff.source.lower() + else ("s__", "SOURCE") + ) + target_prefix, target_display = ( + (f"{target_name}__", target_name.upper()) + if target_name.lower() != row_diff.target.lower() + else ("t__", "TARGET") + ) + + for column in row_diff.joined_sample.columns: + if column.lower().startswith(source_prefix.lower()): + column_name = column[len(source_prefix) :] + + target_column = None + for col in row_diff.joined_sample.columns: + if col.lower() == (target_prefix + column_name).lower(): + target_column = col + break + + if target_column: + columns[column_name] = [column, target_column] + elif not column.lower().startswith(target_prefix.lower()): + keys.append(column) + + column_differences = [] + for column_name, (source_column, target_column) in columns.items(): + column_table = row_diff.joined_sample[keys + [source_column, target_column]] + + # Filter to retain non identical-valued rows + column_table = column_table[ + column_table.apply( + lambda row: not _cells_match(row[source_column], row[target_column]), + axis=1, + ) + ] + + # Rename the column headers for readability + column_table = column_table.rename( + columns={ + source_column: source_display, + target_column: target_display, + } + ) + + if len(column_table) > 0: + for row in column_table.replace({pd.NA: None}).to_dict("records"): + row["__column_name__"] = column_name + row["__source_name__"] = source_display + row["__target_name__"] = target_display + column_differences.append(row) + + return ProcessedSampleData( + column_differences=column_differences, + source_only=row_diff.s_sample.replace({pd.NA: None}).to_dict("records") + if row_diff.s_sample.shape[0] > 0 + else [], + target_only=row_diff.t_sample.replace({pd.NA: None}).to_dict("records") + if row_diff.t_sample.shape[0] > 0 + else [], + ) + + @router.get("") def get_table_diff( source: str, @@ -51,14 +148,24 @@ def get_table_diff( removed=_schema_diff.removed, modified=_schema_diff.modified, ) + + # create a readable column-centric sample data structure + processed_sample_data = _process_sample_data(_row_diff, source, target) + row_diff = RowDiff( source=_row_diff.source, target=_row_diff.target, stats=_row_diff.stats, sample=_row_diff.sample.replace({np.nan: None}).to_dict(), + joined_sample=_row_diff.joined_sample.replace({np.nan: None}).to_dict(), + s_sample=_row_diff.s_sample.replace({np.nan: None}).to_dict(), + t_sample=_row_diff.t_sample.replace({np.nan: None}).to_dict(), + column_stats=_row_diff.column_stats.replace({np.nan: None}).to_dict(), source_count=_row_diff.source_count, target_count=_row_diff.target_count, count_pct_change=_row_diff.count_pct_change, + decimals=getattr(_row_diff, "decimals", 3), + processed_sample_data=processed_sample_data, ) s_index, t_index, _ = diff.key_columns diff --git a/web/server/models.py b/web/server/models.py index d193fa5f07..d26848e068 100644 --- a/web/server/models.py +++ b/web/server/models.py @@ -392,24 +392,47 @@ def validate_schema( v: t.Union[ t.Dict[str, exp.DataType], t.List[t.Tuple[str, exp.DataType]], + t.Dict[str, t.Tuple[exp.DataType, exp.DataType]], t.Dict[str, str], ], + info: ValidationInfo, ) -> t.Dict[str, str]: if isinstance(v, dict): - return {k: str(v) for k, v in v.items()} + # Handle modified field which has tuples of (source_type, target_type) + if info.field_name == "modified" and any(isinstance(val, tuple) for val in v.values()): + return { + k: f"{str(val[0])} → {str(val[1])}" + for k, val in v.items() + if isinstance(val, tuple) + and isinstance(val[0], exp.DataType) + and isinstance(val[1], exp.DataType) + } + return {k: str(val) for k, val in v.items()} if isinstance(v, list): - return {k: str(v) for k, v in v} + return {k: str(val) for k, val in v} return v +class ProcessedSampleData(PydanticModel): + column_differences: t.List[t.Dict[str, t.Any]] + source_only: t.List[t.Dict[str, t.Any]] + target_only: t.List[t.Dict[str, t.Any]] + + class RowDiff(PydanticModel): source: str target: str stats: t.Dict[str, float] sample: t.Dict[str, t.Any] + joined_sample: t.Dict[str, t.Any] + s_sample: t.Dict[str, t.Any] + t_sample: t.Dict[str, t.Any] + column_stats: t.Dict[str, t.Any] source_count: int target_count: int count_pct_change: float + decimals: int + processed_sample_data: t.Optional[ProcessedSampleData] = None class TableDiff(PydanticModel):