[wip] cot api and rendering

src/vs/base/common/marshallingIds.ts

@@ -25,6 +25,7 @@ export const enum MarshalledId {
 	ChatViewContext,
 	LanguageModelToolResult,
 	LanguageModelTextPart,
+	LanguageModelThinkingPart,
 	LanguageModelPromptTsxPart,
 	LanguageModelDataPart,
 	ChatSessionContext,

src/vs/platform/extensions/common/extensionsApiProposals.ts

@@ -251,6 +251,10 @@ const _allApiProposals = {
 	languageModelSystem: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelSystem.d.ts',
 	},
+	languageModelThinking: {
+		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelThinkingPart.d.ts',
+		version: 1
+	},
 	languageStatusText: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageStatusText.d.ts',
 	},

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -1864,6 +1864,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			LanguageModelToolResultPart2: extHostTypes.LanguageModelToolResultPart2,
 			LanguageModelTextPart: extHostTypes.LanguageModelTextPart,
 			LanguageModelToolCallPart: extHostTypes.LanguageModelToolCallPart,
+			LanguageModelThinkingPart: extHostTypes.LanguageModelThinkingPart,
 			LanguageModelError: extHostTypes.LanguageModelError,
 			LanguageModelToolResult: extHostTypes.LanguageModelToolResult,
 			LanguageModelToolResult2: extHostTypes.LanguageModelToolResult2,

src/vs/workbench/api/common/extHostLanguageModels.ts

@@ -40,13 +40,13 @@ type LanguageModelProviderData = {
 
 class LanguageModelResponseStream {
 
-	readonly stream = new AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelToolCallPart>();
+	readonly stream = new AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelThinkingPart | vscode.LanguageModelToolCallPart>();
 
 	constructor(
 		readonly option: number,
-		stream?: AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelToolCallPart>
+		stream?: AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelThinkingPart | vscode.LanguageModelToolCallPart>
 	) {
-		this.stream = stream ?? new AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelToolCallPart>();
+		this.stream = stream ?? new AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelThinkingPart | vscode.LanguageModelToolCallPart>();
 	}
 }
 
@@ -55,7 +55,7 @@ class LanguageModelResponse {
 	readonly apiObject: vscode.LanguageModelChatResponse;
 
 	private readonly _responseStreams = new Map<number, LanguageModelResponseStream>();
-	private readonly _defaultStream = new AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelToolCallPart>();
+	private readonly _defaultStream = new AsyncIterableSource<vscode.LanguageModelTextPart | vscode.LanguageModelThinkingPart | vscode.LanguageModelToolCallPart>();
 	private _isDone: boolean = false;
 
 	constructor() {
@@ -93,13 +93,15 @@ class LanguageModelResponse {
 			return;
 		}
 
-		const partsByIndex = new Map<number, (vscode.LanguageModelTextPart | vscode.LanguageModelToolCallPart)[]>();
+		const partsByIndex = new Map<number, (vscode.LanguageModelTextPart | vscode.LanguageModelThinkingPart | vscode.LanguageModelToolCallPart)[]>();
 
 		for (const fragment of Iterable.wrap(fragments)) {
 
-			let out: vscode.LanguageModelTextPart | vscode.LanguageModelToolCallPart;
+			let out: vscode.LanguageModelTextPart | vscode.LanguageModelThinkingPart | vscode.LanguageModelToolCallPart;
 			if (fragment.part.type === 'text') {
 				out = new extHostTypes.LanguageModelTextPart(fragment.part.value);
+			} else if (fragment.part.type === 'thinking') {
+				out = new extHostTypes.LanguageModelThinkingPart(fragment.part.value, fragment.part.id, fragment.part.metadata);
 			} else if (fragment.part.type === 'data') {
 				out = new extHostTypes.LanguageModelTextPart('');
 			} else {
@@ -299,6 +301,8 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
 				part = { type: 'text', value: fragment.part.value };
 			} else if (fragment.part instanceof extHostTypes.LanguageModelDataPart) {
 				part = { type: 'data', value: { mimeType: fragment.part.mimeType as ChatImageMimeType, data: VSBuffer.wrap(fragment.part.data) } };
+			} else if (fragment.part instanceof extHostTypes.LanguageModelThinkingPart) {
+				part = { type: 'thinking', value: fragment.part.value, id: fragment.part.id, metadata: fragment.part.metadata };
 			}
 
 			if (!part) {

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -3127,6 +3127,8 @@ export namespace ChatAgentResult {
 				return new types.LanguageModelToolResult(cloneAndChange(value.content, reviveMetadata));
 			} else if (value.$mid === MarshalledId.LanguageModelTextPart) {
 				return new types.LanguageModelTextPart(value.value);
+			} else if (value.$mid === MarshalledId.LanguageModelThinkingPart) {
+				return new types.LanguageModelThinkingPart(value.value, value.id, value.metadata);
 			} else if (value.$mid === MarshalledId.LanguageModelPromptTsxPart) {
 				return new types.LanguageModelPromptTsxPart(value.value);
 			}

src/vs/workbench/api/common/extHostTypes.ts

@@ -5017,6 +5017,28 @@ export enum ChatImageMimeType {
 	BMP = 'image/bmp',
 }
 
+export class LanguageModelThinkingPart implements vscode.LanguageModelThinkingPart {
+	value: string;
+	id?: string;
+	metadata?: string;
+
+	constructor(value: string, id?: string, metadata?: string) {
+		this.value = value;
+		this.id = id;
+		this.metadata = metadata;
+	}
+
+	toJSON() {
+		return {
+			$mid: MarshalledId.LanguageModelThinkingPart,
+			value: this.value,
+			id: this.id,
+			metadata: this.metadata,
+		};
+	}
+}
+
+
 
 export class LanguageModelPromptTsxPart {
 	value: unknown;

src/vs/workbench/contrib/chat/browser/chatContentParts/chatThinkingContentPart.ts

@@ -0,0 +1,92 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as dom from '../../../../../base/browser/dom.js';
+import { Emitter } from '../../../../../base/common/event.js';
+import { Disposable, IDisposable } from '../../../../../base/common/lifecycle.js';
+import { IChatThinkingPart } from '../../common/chatService.js';
+import { IChatContentPartRenderContext, IChatContentPart } from './chatContentParts.js';
+
+export class ChatThinkingContentPart extends Disposable implements IChatContentPart {
+
+	readonly domNode: HTMLElement;
+	public readonly codeblocks: undefined;
+	public readonly codeblocksPartId: undefined;
+
+	private readonly _onDidChangeHeight = new Emitter<void>();
+	readonly onDidChangeHeight = this._onDidChangeHeight.event;
+
+	addDisposable<T extends IDisposable>(disposable: T): T {
+		return this._register(disposable);
+	}
+
+	constructor(
+		private readonly content: IChatThinkingPart,
+		_context: IChatContentPartRenderContext,
+	) {
+		super();
+
+		this.domNode = dom.$('div.thinking-content');
+		this.domNode.classList.add('chat-thinking');
+
+		const thinkingContainer = dom.$('div.thinking-container');
+		dom.append(this.domNode, thinkingContainer);
+
+		const iconContainer = dom.$('div.thinking-icon');
+		dom.append(thinkingContainer, iconContainer);
+		dom.append(iconContainer, dom.$('span.codicon.codicon-lightbulb'));
+
+		// thinking content
+		const textContainer = dom.$('div.thinking-text');
+		dom.append(thinkingContainer, textContainer);
+		textContainer.textContent = content.value;
+
+		if (content.metadata) {
+			const metadataContainer = dom.$('div.thinking-metadata');
+			dom.append(thinkingContainer, metadataContainer);
+			metadataContainer.textContent = content.metadata;
+		}
+	}
+
+	hasSameContent(other: IChatThinkingPart): boolean {
+		return other.kind === 'thinking' &&
+			other.value === this.content.value &&
+			other.id === this.content.id &&
+			other.metadata === this.content.metadata;
+	}
+
+	update(newContent: IChatThinkingPart): void {
+		let contentChanged = false;
+
+		if (this.content.value !== newContent.value) {
+			const textContainer = this.domNode.querySelector('.thinking-text') as HTMLElement;
+			if (textContainer) {
+				textContainer.textContent = newContent.value;
+				contentChanged = true;
+			}
+		}
+
+		if (this.content.metadata !== newContent.metadata && newContent.metadata) {
+			let metadataContainer = this.domNode.querySelector('.thinking-metadata') as HTMLElement;
+			if (!metadataContainer && newContent.metadata) {
+				const thinkingContainer = this.domNode.querySelector('.thinking-container') as HTMLElement;
+				if (thinkingContainer) {
+					metadataContainer = dom.$('div.thinking-metadata');
+					dom.append(thinkingContainer, metadataContainer);
+					contentChanged = true;
+				}
+			}
+
+			if (metadataContainer) {
+				metadataContainer.textContent = newContent.metadata;
+				contentChanged = true;
+			}
+		}
+
+		if (contentChanged) {
+			this._onDidChangeHeight.fire();
+		}
+	}
+}

src/vs/workbench/contrib/chat/browser/chatListRenderer.ts

@@ -49,7 +49,7 @@ import { IChatAgentMetadata } from '../common/chatAgents.js';
 import { ChatContextKeys } from '../common/chatContextKeys.js';
 import { IChatTextEditGroup } from '../common/chatModel.js';
 import { chatSubcommandLeader } from '../common/chatParserTypes.js';
-import { ChatAgentVoteDirection, ChatAgentVoteDownReason, ChatErrorLevel, IChatConfirmation, IChatContentReference, IChatElicitationRequest, IChatExtensionsContent, IChatFollowup, IChatMarkdownContent, IChatTask, IChatTaskSerialized, IChatToolInvocation, IChatToolInvocationSerialized, IChatTreeData, IChatUndoStop } from '../common/chatService.js';
+import { ChatAgentVoteDirection, ChatAgentVoteDownReason, ChatErrorLevel, IChatConfirmation, IChatContentReference, IChatElicitationRequest, IChatExtensionsContent, IChatFollowup, IChatMarkdownContent, IChatTask, IChatTaskSerialized, IChatThinkingPart, IChatToolInvocation, IChatToolInvocationSerialized, IChatTreeData, IChatUndoStop } from '../common/chatService.js';
 import { IChatCodeCitations, IChatErrorDetailsPart, IChatReferences, IChatRendererContent, IChatRequestViewModel, IChatResponseViewModel, IChatViewModel, IChatWorkingProgress, isRequestVM, isResponseVM } from '../common/chatViewModel.js';
 import { getNWords } from '../common/chatWordCounter.js';
 import { CodeBlockModelCollection } from '../common/codeBlockModelCollection.js';
@@ -81,6 +81,7 @@ import { ChatCodeBlockContentProvider, CodeBlockPart } from './codeBlockPart.js'
 import { canceledName } from '../../../../base/common/errors.js';
 import { IChatRequestVariableEntry } from '../common/chatVariableEntries.js';
 import { ChatElicitationContentPart } from './chatContentParts/chatElicitationContentPart.js';
+import { ChatThinkingContentPart } from './chatContentParts/chatThinkingContentPart.js';
 import { alert } from '../../../../base/browser/ui/aria/aria.js';
 import { CodiconActionViewItem } from '../../notebook/browser/view/cellParts/cellActionView.js';
 
@@ -1084,6 +1085,8 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer<Ch
 				return this.renderChatErrorDetails(context, content, templateData);
 			} else if (content.kind === 'elicitation') {
 				return this.renderElicitation(context, content, templateData);
+			} else if (content.kind === 'thinking') {
+				return this.renderThinking(context, content, templateData);
 			}
 
 			return this.renderNoContent(other => content.kind === other.kind);
@@ -1265,6 +1268,12 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer<Ch
 		return part;
 	}
 
+	private renderThinking(context: IChatContentPartRenderContext, thinking: IChatThinkingPart, templateData: IChatListItemTemplate): IChatContentPart {
+		const part = this.instantiationService.createInstance(ChatThinkingContentPart, thinking, context);
+		part.addDisposable(part.onDidChangeHeight(() => this.updateItemHeight(templateData)));
+		return part;
+	}
+
 	private renderAttachments(variables: IChatRequestVariableEntry[], contentReferences: ReadonlyArray<IChatContentReference> | undefined, templateData: IChatListItemTemplate) {
 		return this.instantiationService.createInstance(ChatAttachmentsContentPart, variables, contentReferences, undefined);
 	}

src/vs/workbench/contrib/chat/common/chatModel.ts

@@ -25,7 +25,7 @@ import { CellUri, ICellEditOperation } from '../../notebook/common/notebookCommo
 import { IChatAgentCommand, IChatAgentData, IChatAgentResult, IChatAgentService, reviveSerializedAgent } from './chatAgents.js';
 import { IChatEditingService, IChatEditingSession } from './chatEditingService.js';
 import { ChatRequestTextPart, IParsedChatRequest, reviveParsedChatRequest } from './chatParserTypes.js';
-import { ChatAgentVoteDirection, ChatAgentVoteDownReason, IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatEditingSessionAction, IChatElicitationRequest, IChatExtensionsContent, IChatFollowup, IChatLocationData, IChatMarkdownContent, IChatNotebookEdit, IChatPrepareToolInvocationPart, IChatProgress, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatResponseProgressFileTreeData, IChatTask, IChatTaskSerialized, IChatTextEdit, IChatToolInvocation, IChatToolInvocationSerialized, IChatTreeData, IChatUndoStop, IChatUsedContext, IChatWarningMessage, isIUsedContext } from './chatService.js';
+import { ChatAgentVoteDirection, ChatAgentVoteDownReason, IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatEditingSessionAction, IChatElicitationRequest, IChatExtensionsContent, IChatFollowup, IChatLocationData, IChatMarkdownContent, IChatNotebookEdit, IChatPrepareToolInvocationPart, IChatProgress, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatResponseProgressFileTreeData, IChatTask, IChatTaskSerialized, IChatTextEdit, IChatThinkingPart, IChatToolInvocation, IChatToolInvocationSerialized, IChatTreeData, IChatUndoStop, IChatUsedContext, IChatWarningMessage, isIUsedContext } from './chatService.js';
 import { IChatRequestVariableEntry } from './chatVariableEntries.js';
 import { ChatAgentLocation, ChatModeKind } from './constants.js';
 
@@ -115,7 +115,8 @@ export type IChatProgressHistoryResponseContent =
 	| IChatTextEditGroup
 	| IChatNotebookEditGroup
 	| IChatConfirmation
-	| IChatExtensionsContent;
+	| IChatExtensionsContent
+	| IChatThinkingPart;
 
 /**
  * "Normal" progress kinds that are rendered as parts of the stream of content.
@@ -137,6 +138,21 @@ export function toChatHistoryContent(content: ReadonlyArray<IChatProgressRespons
 	return content.filter(isChatProgressHistoryResponseContent);
 }
 
+/**
+ * Extract thinking tokens from chat response content
+ */
+export function getThinkingTokens(content: ReadonlyArray<IChatProgressResponseContent>): IChatThinkingPart[] {
+	return content.filter((part): part is IChatThinkingPart => part.kind === 'thinking');
+}
+
+/**
+ * Get thinking tokens from a chat response model
+ */
+export function getResponseThinkingTokens(response: IChatResponseModel): IChatThinkingPart[] {
+	return getThinkingTokens(response.entireResponse.value);
+}
+
+
 export type IChatProgressRenderableResponseContent = Exclude<IChatProgressResponseContent, IChatContentInlineReference | IChatAgentMarkdownContentWithVulnerability | IChatResponseCodeblockUriPart>;
 
 export interface IResponse {
@@ -356,6 +372,7 @@ class AbstractResponse implements IResponse {
 				case 'undoStop':
 				case 'prepareToolInvocation':
 				case 'elicitation':
+				case 'thinking':
 					// Ignore
 					continue;
 				case 'inlineReference':
@@ -1682,6 +1699,13 @@ export class ChatModel extends Disposable implements IChatModel {
 								return item.treeData;
 							} else if (item.kind === 'markdownContent') {
 								return item.content;
+							} else if (item.kind === 'thinking') {
+								return {
+									kind: 'thinking',
+									value: item.value,
+									id: item.id,
+									metadata: item.metadata
+								};
 							} else {
 								return item as any; // TODO
 							}

src/vs/workbench/contrib/chat/common/chatService.ts

@@ -240,6 +240,14 @@ export interface IChatElicitationRequest {
 	reject(): Promise<void>;
 }
 
+export interface IChatThinkingPart {
+	kind: 'thinking';
+	value: string;
+	id?: string;
+	metadata?: string;
+}
+
+
 export interface IChatTerminalToolInvocationData {
 	kind: 'terminal';
 	command: string;
@@ -352,6 +360,7 @@ export type IChatProgress =
 	| IChatExtensionsContent
 	| IChatUndoStop
 	| IChatPrepareToolInvocationPart
+	| IChatThinkingPart
 	| IChatTaskSerialized
 	| IChatElicitationRequest;
 

src/vs/workbench/contrib/chat/common/languageModels.ts

@@ -109,7 +109,14 @@ export interface IChatResponseToolUsePart {
 	parameters: any;
 }
 
-export type IChatResponsePart = IChatResponseTextPart | IChatResponseToolUsePart | IChatResponseDataPart;
+export interface IChatResponseThinkingPart {
+	type: 'thinking';
+	value: string;
+	id?: string;
+	metadata?: string;
+}
+
+export type IChatResponsePart = IChatResponseTextPart | IChatResponseToolUsePart | IChatResponseDataPart | IChatResponseThinkingPart;
 
 export interface IChatResponseFragment {
 	index: number;