diff --git a/js/plugins/vertexai/src/modelgarden/index.ts b/js/plugins/vertexai/src/modelgarden/index.ts index 41d03c6888..d9fe910dc1 100644 --- a/js/plugins/vertexai/src/modelgarden/index.ts +++ b/js/plugins/vertexai/src/modelgarden/index.ts @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,58 +14,6 @@ * limitations under the License. */ -import type { Genkit } from 'genkit'; -import { genkitPlugin, type GenkitPlugin } from 'genkit/plugin'; -import { getDerivedParams } from '../common/index.js'; -import { SUPPORTED_ANTHROPIC_MODELS, anthropicModel } from './anthropic.js'; -import { SUPPORTED_MISTRAL_MODELS, mistralModel } from './mistral.js'; -import { - SUPPORTED_OPENAI_FORMAT_MODELS, - modelGardenOpenaiCompatibleModel, -} from './model_garden.js'; -import type { PluginOptions } from './types.js'; - -/** - * Add Google Cloud Vertex AI Rerankers API to Genkit. - */ -export function vertexAIModelGarden(options: PluginOptions): GenkitPlugin { - return genkitPlugin('vertexAIModelGarden', async (ai: Genkit) => { - const { projectId, location, authClient } = await getDerivedParams(options); - - options.models.forEach((m) => { - const anthropicEntry = Object.entries(SUPPORTED_ANTHROPIC_MODELS).find( - ([_, value]) => value.name === m.name - ); - if (anthropicEntry) { - anthropicModel(ai, anthropicEntry[0], projectId, location); - return; - } - const mistralEntry = Object.entries(SUPPORTED_MISTRAL_MODELS).find( - ([_, value]) => value.name === m.name - ); - if (mistralEntry) { - mistralModel(ai, mistralEntry[0], projectId, location); - return; - } - const openaiModel = Object.entries(SUPPORTED_OPENAI_FORMAT_MODELS).find( - ([_, value]) => value.name === m.name - ); - if (openaiModel) { - modelGardenOpenaiCompatibleModel( - ai, - openaiModel[0], - projectId, - location, - authClient, - options.openAiBaseUrlTemplate - ); - return; - } - throw new Error(`Unsupported model garden model: ${m.name}`); - }); - }); -} - export { claude35Sonnet, claude35SonnetV2, @@ -75,7 +23,16 @@ export { claudeOpus4, claudeOpus41, claudeSonnet4, -} from './anthropic.js'; -export { codestral, mistralLarge, mistralNemo } from './mistral.js'; -export { llama3, llama31, llama32 } from './model_garden.js'; -export type { PluginOptions }; + codestral, + llama3, + llama31, + llama32, + mistralLarge, + mistralNemo, + vertexAIModelGarden, +} from './legacy/index.js'; +export { + vertexModelGarden, + type PluginOptions, + type VertexModelGardenPlugin, +} from './v2/index.js'; diff --git a/js/plugins/vertexai/src/modelgarden/anthropic.ts b/js/plugins/vertexai/src/modelgarden/legacy/anthropic.ts similarity index 95% rename from js/plugins/vertexai/src/modelgarden/anthropic.ts rename to js/plugins/vertexai/src/modelgarden/legacy/anthropic.ts index 848ee1ed97..81b8f32fad 100644 --- a/js/plugins/vertexai/src/modelgarden/anthropic.ts +++ b/js/plugins/vertexai/src/modelgarden/legacy/anthropic.ts @@ -45,12 +45,15 @@ import { modelRef, type ModelAction, } from 'genkit/model'; -import { getGenkitClientHeader } from '../common/index.js'; +import { getGenkitClientHeader } from '../../common/index.js'; export const AnthropicConfigSchema = GenerationCommonConfigSchema.extend({ location: z.string().optional(), }); +/** + * @deprecated + */ export const claude35SonnetV2 = modelRef({ name: 'vertexai/claude-3-5-sonnet-v2', info: { @@ -67,6 +70,9 @@ export const claude35SonnetV2 = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated + */ export const claude35Sonnet = modelRef({ name: 'vertexai/claude-3-5-sonnet', info: { @@ -83,6 +89,9 @@ export const claude35Sonnet = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated + */ export const claude3Sonnet = modelRef({ name: 'vertexai/claude-3-sonnet', info: { @@ -99,6 +108,9 @@ export const claude3Sonnet = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated Please use vertexModelGarden.model('claude-3-5-haiku@20241022') + */ export const claude3Haiku = modelRef({ name: 'vertexai/claude-3-haiku', info: { @@ -115,6 +127,9 @@ export const claude3Haiku = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated + */ export const claude3Opus = modelRef({ name: 'vertexai/claude-3-opus', info: { @@ -131,6 +146,9 @@ export const claude3Opus = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated please use vertexModelGarden.model('claude-sonnet-4@20250514') + */ export const claudeSonnet4 = modelRef({ name: 'vertexai/claude-sonnet-4', info: { @@ -147,6 +165,9 @@ export const claudeSonnet4 = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated please use vertexModelGarden.model('claude-opus-4@20250514') + */ export const claudeOpus4 = modelRef({ name: 'vertexai/claude-opus-4', info: { @@ -163,6 +184,9 @@ export const claudeOpus4 = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated please use vertexModelGarden.model('claude-opus-4-1@20250805') + */ export const claudeOpus41 = modelRef({ name: 'vertexai/claude-opus-4-1', info: { @@ -179,6 +203,9 @@ export const claudeOpus41 = modelRef({ configSchema: AnthropicConfigSchema, }); +/** + * @deprecated + */ export const SUPPORTED_ANTHROPIC_MODELS: Record< string, ModelReference @@ -193,6 +220,9 @@ export const SUPPORTED_ANTHROPIC_MODELS: Record< 'claude-opus-4-1': claudeOpus41, }; +/** + * @deprecated + */ export function toAnthropicRequest( model: string, input: GenerateRequest @@ -339,6 +369,9 @@ function fromAnthropicPart(part: AnthropicContent): Part { } // Converts an Anthropic response to a Genkit response. +/** + * @deprecated + */ export function fromAnthropicResponse( input: GenerateRequest, response: Message @@ -428,6 +461,9 @@ function toAnthropicToolResponse(part: Part): ToolResultBlockParam { }; } +/** + * @deprecated + */ export function anthropicModel( ai: Genkit, modelName: string, diff --git a/js/plugins/vertexai/src/modelgarden/legacy/index.ts b/js/plugins/vertexai/src/modelgarden/legacy/index.ts new file mode 100644 index 0000000000..4b5b2c53f2 --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/legacy/index.ts @@ -0,0 +1,82 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Genkit } from 'genkit'; +import { genkitPlugin, type GenkitPlugin } from 'genkit/plugin'; +import { getDerivedParams } from '../../common/index.js'; +import { SUPPORTED_ANTHROPIC_MODELS, anthropicModel } from './anthropic.js'; +import { SUPPORTED_MISTRAL_MODELS, mistralModel } from './mistral.js'; +import { + SUPPORTED_OPENAI_FORMAT_MODELS, + modelGardenOpenaiCompatibleModel, +} from './model_garden.js'; +import type { PluginOptions } from './types.js'; + +/** + * Add Google Cloud Vertex AI Rerankers API to Genkit. + * @deprecated Please use vertexModelGarden + */ +export function vertexAIModelGarden(options: PluginOptions): GenkitPlugin { + return genkitPlugin('vertexAIModelGarden', async (ai: Genkit) => { + const { projectId, location, authClient } = await getDerivedParams(options); + + options.models.forEach((m) => { + const anthropicEntry = Object.entries(SUPPORTED_ANTHROPIC_MODELS).find( + ([_, value]) => value.name === m.name + ); + if (anthropicEntry) { + anthropicModel(ai, anthropicEntry[0], projectId, location); + return; + } + const mistralEntry = Object.entries(SUPPORTED_MISTRAL_MODELS).find( + ([_, value]) => value.name === m.name + ); + if (mistralEntry) { + mistralModel(ai, mistralEntry[0], projectId, location); + return; + } + const openaiModel = Object.entries(SUPPORTED_OPENAI_FORMAT_MODELS).find( + ([_, value]) => value.name === m.name + ); + if (openaiModel) { + modelGardenOpenaiCompatibleModel( + ai, + openaiModel[0], + projectId, + location, + authClient, + options.openAiBaseUrlTemplate + ); + return; + } + throw new Error(`Unsupported model garden model: ${m.name}`); + }); + }); +} + +export { + claude35Sonnet, + claude35SonnetV2, + claude3Haiku, + claude3Opus, + claude3Sonnet, + claudeOpus4, + claudeOpus41, + claudeSonnet4, +} from './anthropic.js'; +export { codestral, mistralLarge, mistralNemo } from './mistral.js'; +export { llama3, llama31, llama32 } from './model_garden.js'; +//export type { PluginOptions }; // Same one will be exported by v2 now diff --git a/js/plugins/vertexai/src/modelgarden/mistral.ts b/js/plugins/vertexai/src/modelgarden/legacy/mistral.ts similarity index 97% rename from js/plugins/vertexai/src/modelgarden/mistral.ts rename to js/plugins/vertexai/src/modelgarden/legacy/mistral.ts index df342434c0..c5b7ba29f8 100644 --- a/js/plugins/vertexai/src/modelgarden/mistral.ts +++ b/js/plugins/vertexai/src/modelgarden/legacy/mistral.ts @@ -46,10 +46,11 @@ import { modelRef, type ModelAction, } from 'genkit/model'; -import { getGenkitClientHeader } from '../common/index.js'; +import { getGenkitClientHeader } from '../../common/index.js'; /** * See https://docs.mistral.ai/api/#tag/chat/operation/chat_completion_v1_chat_completions_post + * @deprecated */ export const MistralConfigSchema = GenerationCommonConfigSchema.extend({ // TODO: Update this with all the parameters in @@ -63,6 +64,9 @@ export const MistralConfigSchema = GenerationCommonConfigSchema.extend({ .optional(), }); +/** + * @deprecated + */ export const mistralLarge = modelRef({ name: 'vertexai/mistral-large', info: { @@ -79,6 +83,9 @@ export const mistralLarge = modelRef({ configSchema: MistralConfigSchema, }); +/** + * @deprecated + */ export const mistralNemo = modelRef({ name: 'vertexai/mistral-nemo', info: { @@ -95,6 +102,9 @@ export const mistralNemo = modelRef({ configSchema: MistralConfigSchema, }); +/** + * @deprecated + */ export const codestral = modelRef({ name: 'vertexai/codestral', info: { @@ -111,6 +121,9 @@ export const codestral = modelRef({ configSchema: MistralConfigSchema, }); +/** + * @deprecated + */ export const SUPPORTED_MISTRAL_MODELS: Record< string, ModelReference @@ -152,6 +165,9 @@ function toMistralToolRequest(toolRequest: Record): FunctionCall { }; } +/** + * @deprecated + */ export function toMistralRequest( model: string, input: GenerateRequest @@ -278,7 +294,10 @@ function fromMistralMessage(message: AssistantMessage): Part[] { return parts; } -// Maps Mistral finish reasons to Genkit finish reasons +/** + * Maps Mistral finish reasons to Genkit finish reasons + * @deprecated + */ export function fromMistralFinishReason( reason: ChatCompletionChoiceFinishReason | undefined ): 'length' | 'unknown' | 'stop' | 'blocked' | 'other' { @@ -297,7 +316,10 @@ export function fromMistralFinishReason( } } -// Converts a Mistral response to a Genkit response +/** + * Converts a Mistral response to a Genkit response + * @deprecated + */ export function fromMistralResponse( _input: GenerateRequest, response: ChatCompletionResponse @@ -329,6 +351,7 @@ export function fromMistralResponse( }; } +/** @deprecated */ export function mistralModel( ai: Genkit, modelName: string, @@ -467,6 +490,7 @@ function validateToolSequence(messages: MistralMessage[]) { }); } +/** @deprecated */ export function fromMistralCompletionChunk(chunk: CompletionChunk): Part[] { if (!chunk.choices?.[0]?.delta) return []; diff --git a/js/plugins/vertexai/src/modelgarden/model_garden.ts b/js/plugins/vertexai/src/modelgarden/legacy/model_garden.ts similarity index 95% rename from js/plugins/vertexai/src/modelgarden/model_garden.ts rename to js/plugins/vertexai/src/modelgarden/legacy/model_garden.ts index 19c7075551..81506b85bd 100644 --- a/js/plugins/vertexai/src/modelgarden/model_garden.ts +++ b/js/plugins/vertexai/src/modelgarden/legacy/model_garden.ts @@ -18,16 +18,18 @@ import { z, type Genkit, type ModelReference } from 'genkit'; import { modelRef, type GenerateRequest, type ModelAction } from 'genkit/model'; import type { GoogleAuth } from 'google-auth-library'; import OpenAI from 'openai'; -import { getGenkitClientHeader } from '../common/index.js'; +import { getGenkitClientHeader } from '../../common/index.js'; import { OpenAIConfigSchema, openaiCompatibleModel, } from './openai_compatibility.js'; +/** @deprecated */ export const ModelGardenModelConfigSchema = OpenAIConfigSchema.extend({ location: z.string().optional(), }); +/** @deprecated */ export const llama31 = modelRef({ name: 'vertexai/llama-3.1', info: { @@ -48,6 +50,7 @@ export const llama31 = modelRef({ version: 'meta/llama3-405b-instruct-maas', }) as ModelReference; +/** @deprecated */ export const llama32 = modelRef({ name: 'vertexai/llama-3.2', info: { @@ -85,12 +88,14 @@ export const llama3 = modelRef({ version: 'meta/llama3-405b-instruct-maas', }) as ModelReference; +/** @deprecated */ export const SUPPORTED_OPENAI_FORMAT_MODELS = { 'llama3-405b': llama3, 'llama-3.1': llama31, 'llama-3.2': llama32, }; +/** @deprecated */ export function modelGardenOpenaiCompatibleModel( ai: Genkit, name: string, diff --git a/js/plugins/vertexai/src/modelgarden/openai_compatibility.ts b/js/plugins/vertexai/src/modelgarden/legacy/openai_compatibility.ts similarity index 98% rename from js/plugins/vertexai/src/modelgarden/openai_compatibility.ts rename to js/plugins/vertexai/src/modelgarden/legacy/openai_compatibility.ts index d21e596f59..1a4d501378 100644 --- a/js/plugins/vertexai/src/modelgarden/openai_compatibility.ts +++ b/js/plugins/vertexai/src/modelgarden/legacy/openai_compatibility.ts @@ -44,6 +44,7 @@ import type { /** * See https://platform.openai.com/docs/api-reference/chat/create. + * @deprecated */ export const OpenAIConfigSchema = GenerationCommonConfigSchema.extend({ // TODO: topK is not supported and some of the other common config options @@ -120,6 +121,7 @@ export const OpenAIConfigSchema = GenerationCommonConfigSchema.extend({ .optional(), }); +/** @deprecated */ export function toOpenAIRole(role: Role): ChatCompletionRole { switch (role) { case 'user': @@ -145,6 +147,7 @@ function toOpenAiTool(tool: ToolDefinition): ChatCompletionTool { }; } +/** @deprecated */ export function toOpenAiTextAndMedia(part: Part): ChatCompletionContentPart { if (part.text) { return { @@ -164,6 +167,7 @@ export function toOpenAiTextAndMedia(part: Part): ChatCompletionContentPart { ); } +/** @deprecated */ export function toOpenAiMessages( messages: MessageData[] ): ChatCompletionMessageParam[] { @@ -243,6 +247,7 @@ const finishReasonMap: Record< content_filter: 'blocked', }; +/** @deprecated */ export function fromOpenAiToolCall( toolCall: | ChatCompletionMessageToolCall @@ -263,6 +268,7 @@ export function fromOpenAiToolCall( }; } +/** @deprecated */ export function fromOpenAiChoice( choice: ChatCompletion.Choice, jsonMode = false @@ -287,6 +293,7 @@ export function fromOpenAiChoice( }; } +/** @deprecated */ export function fromOpenAiChunkChoice( choice: ChatCompletionChunk.Choice, jsonMode = false @@ -311,6 +318,7 @@ export function fromOpenAiChunkChoice( }; } +/** @deprecated */ export function toRequestBody( model: ModelReference, request: GenerateRequest @@ -363,6 +371,7 @@ export function toRequestBody( return body; } +/** @deprecated */ export function openaiCompatibleModel( ai: Genkit, model: ModelReference, diff --git a/js/plugins/vertexai/src/modelgarden/legacy/types.ts b/js/plugins/vertexai/src/modelgarden/legacy/types.ts new file mode 100644 index 0000000000..29ead9c851 --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/legacy/types.ts @@ -0,0 +1,39 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ModelReference } from 'genkit'; +import type { CommonPluginOptions } from '../../common/types.js'; + +/** + * Evaluation metric config. Use `metricSpec` to define the behavior of the metric. + * The value of `metricSpec` will be included in the request to the API. See the API documentation + * for details on the possible values of `metricSpec` for each metric. + * https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation#parameter-list + */ + +/** + * Options specific to Model Garden configuration + * @deprecated + * */ +export interface ModelGardenOptions { + models: ModelReference[]; + openAiBaseUrlTemplate?: string; +} + +/** @deprecated */ +export interface PluginOptions + extends CommonPluginOptions, + ModelGardenOptions {} diff --git a/js/plugins/vertexai/src/modelgarden/v2/anthropic.ts b/js/plugins/vertexai/src/modelgarden/v2/anthropic.ts new file mode 100644 index 0000000000..ce620f598a --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/v2/anthropic.ts @@ -0,0 +1,430 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + ContentBlock as AnthropicContent, + ImageBlockParam, + Message, + MessageCreateParamsBase, + MessageParam, + TextBlock, + TextBlockParam, + TextDelta, + Tool, + ToolResultBlockParam, + ToolUseBlock, + ToolUseBlockParam, +} from '@anthropic-ai/sdk/resources/messages'; +import { AnthropicVertex } from '@anthropic-ai/vertex-sdk'; +import { + ActionMetadata, + z, + type GenerateRequest, + type Part as GenkitPart, + type MessageData, + type ModelReference, + type ModelResponseData, + type Part, +} from 'genkit'; +import { + GenerationCommonConfigSchema, + ModelInfo, + getBasicUsageStats, + modelRef, + type ModelAction, +} from 'genkit/model'; +import { model as pluginModel } from 'genkit/plugin'; +import { getGenkitClientHeader } from '../../common/index.js'; +import { PluginOptions } from './types.js'; +import { checkModelName } from './utils.js'; + +export const AnthropicConfigSchema = GenerationCommonConfigSchema.extend({ + location: z.string().optional(), +}).passthrough(); +export type AnthropicConfigSchemaType = typeof AnthropicConfigSchema; +export type AnthropicConfig = z.infer; + +// All the config schema types +type ConfigSchemaType = AnthropicConfigSchemaType; + +function commonRef( + name: string, + info?: ModelInfo, + configSchema: ConfigSchemaType = AnthropicConfigSchema +): ModelReference { + return modelRef({ + name: `vertex-model-garden/${name}`, + configSchema, + info: info ?? { + supports: { + multiturn: true, + media: true, + tools: true, + systemRole: true, + output: ['text'], + }, + }, + }); +} + +export const GENERIC_MODEL = commonRef('anthropic'); + +export const KNOWN_MODELS = { + 'claude-sonnet-4-5@20250929': commonRef('claude-sonnet-4-5@20250929'), + 'claude-sonnet-4@20250514': commonRef('claude-sonnet-4@20250514'), + 'claude-3-7-sonnet@20250219': commonRef('claude-3-7-sonnet@20250219'), + 'claude-opus-4-1@20250805': commonRef('claude-opus-4-1@20250805'), + 'claude-opus-4@20250514': commonRef('claude-opus-4@20250514'), + 'claude-3-5-haiku@20241022': commonRef('claude-3-5-haiku@20241022'), + 'claude-3-haiku@20240307': commonRef('claude-3-haiku@20240307'), +}; +export type KnownModels = keyof typeof KNOWN_MODELS; +export type AnthropicModelName = `claude-${string}`; +export function isAnthropicModelName( + value?: string +): value is AnthropicModelName { + return !!value?.startsWith('claude-'); +} + +export function model( + version: string, + options: AnthropicConfig = {} +): ModelReference { + const name = checkModelName(version); + + return modelRef({ + name: `vertex-model-garden/${name}`, + config: options, + configSchema: AnthropicConfigSchema, + info: { + ...GENERIC_MODEL.info, + }, + }); +} + +export interface ClientOptions { + location: string; // e.g. 'us-central1' or 'global' + projectId: string; +} + +export function listActions(clientOptions: ClientOptions): ActionMetadata[] { + // TODO: figure out where to get the list of models. + return []; +} + +export function listKnownModels( + clientOptions: ClientOptions, + pluginOptions?: PluginOptions +) { + return Object.keys(KNOWN_MODELS).map((name) => + defineModel(name, clientOptions, pluginOptions) + ); +} + +export function defineModel( + name: string, + clientOptions: ClientOptions, + pluginOptions?: PluginOptions +): ModelAction { + const clients: Record = {}; + const clientFactory = (region: string): AnthropicVertex => { + if (!clients[region]) { + clients[region] = new AnthropicVertex({ + region: region, + projectId: clientOptions.projectId, + defaultHeaders: { + 'X-Goog-Api-Client': getGenkitClientHeader(), + }, + }); + } + return clients[region]; + }; + const ref = model(name); + + return pluginModel( + { + name: ref.name, + ...ref.info, + configSchema: ref.configSchema, + }, + async (request, { streamingRequested, sendChunk }) => { + const client = clientFactory( + request.config?.location || clientOptions.location + ); + const modelVersion = checkModelName(ref.name); + const anthropicRequest = toAnthropicRequest(modelVersion, request); + if (!streamingRequested) { + // Non-streaming + const response = await client.messages.create({ + ...anthropicRequest, + stream: false, + }); + return fromAnthropicResponse(request, response); + } else { + // Streaming + const stream = await client.messages.stream(anthropicRequest); + for await (const event of stream) { + if (event.type === 'content_block_delta') { + sendChunk({ + index: 0, + content: [ + { + text: (event.delta as TextDelta).text, + }, + ], + }); + } + } + return fromAnthropicResponse(request, await stream.finalMessage()); + } + } + ); +} + +export function toAnthropicRequest( + model: string, + input: GenerateRequest +): MessageCreateParamsBase { + let system: string | undefined = undefined; + const messages: MessageParam[] = []; + for (const msg of input.messages) { + if (msg.role === 'system') { + system = msg.content + .map((c) => { + if (!c.text) { + throw new Error( + 'Only text context is supported for system messages.' + ); + } + return c.text; + }) + .join(); + } + // If the last message is a tool response, we need to add a user message. + // https://docs.anthropic.com/en/docs/build-with-claude/tool-use#handling-tool-use-and-tool-result-content-blocks + else if (msg.content[msg.content.length - 1].toolResponse) { + messages.push({ + role: 'user', + content: toAnthropicContent(msg.content), + }); + } else { + messages.push({ + role: toAnthropicRole(msg.role), + content: toAnthropicContent(msg.content), + }); + } + } + const request = { + model, + messages, + // https://docs.anthropic.com/claude/docs/models-overview#model-comparison + max_tokens: input.config?.maxOutputTokens ?? 4096, + } as MessageCreateParamsBase; + if (system) { + request['system'] = system; + } + if (input.tools) { + request.tools = input.tools?.map((tool) => { + return { + name: tool.name, + description: tool.description, + input_schema: tool.inputSchema, + }; + }) as Array; + } + if (input.config?.stopSequences) { + request.stop_sequences = input.config?.stopSequences; + } + if (input.config?.temperature) { + request.temperature = input.config?.temperature; + } + if (input.config?.topK) { + request.top_k = input.config?.topK; + } + if (input.config?.topP) { + request.top_p = input.config?.topP; + } + return request; +} + +function toAnthropicContent( + content: GenkitPart[] +): Array< + TextBlockParam | ImageBlockParam | ToolUseBlockParam | ToolResultBlockParam +> { + return content.map((p) => { + if (p.text) { + return { + type: 'text', + text: p.text, + }; + } + if (p.media) { + let b64Data = p.media.url; + if (b64Data.startsWith('data:')) { + b64Data = b64Data.substring(b64Data.indexOf(',')! + 1); + } + + return { + type: 'image', + source: { + type: 'base64', + data: b64Data, + media_type: p.media.contentType as + | 'image/jpeg' + | 'image/png' + | 'image/gif' + | 'image/webp', + }, + }; + } + if (p.toolRequest) { + return toAnthropicToolRequest(p.toolRequest); + } + if (p.toolResponse) { + return toAnthropicToolResponse(p); + } + throw new Error(`Unsupported content type: ${JSON.stringify(p)}`); + }); +} + +function toAnthropicRole(role): 'user' | 'assistant' { + if (role === 'model') { + return 'assistant'; + } + if (role === 'user') { + return 'user'; + } + if (role === 'tool') { + return 'assistant'; + } + throw new Error(`Unsupported role type ${role}`); +} + +function fromAnthropicTextPart(part: TextBlock): Part { + return { + text: part.text, + }; +} + +function fromAnthropicToolCallPart(part: ToolUseBlock): Part { + return { + toolRequest: { + name: part.name, + input: part.input, + ref: part.id, + }, + }; +} + +// Converts an Anthropic part to a Genkit part. +function fromAnthropicPart(part: AnthropicContent): Part { + if (part.type === 'text') return fromAnthropicTextPart(part); + if (part.type === 'tool_use') return fromAnthropicToolCallPart(part); + throw new Error( + 'Part type is unsupported/corrupted. Either data is missing or type cannot be inferred from type.' + ); +} + +// Converts an Anthropic response to a Genkit response. +export function fromAnthropicResponse( + input: GenerateRequest, + response: Message +): ModelResponseData { + const parts = response.content as AnthropicContent[]; + const message: MessageData = { + role: 'model', + content: parts.map(fromAnthropicPart), + }; + return { + message, + finishReason: toGenkitFinishReason( + response.stop_reason as + | 'end_turn' + | 'max_tokens' + | 'stop_sequence' + | 'tool_use' + | null + ), + custom: { + id: response.id, + model: response.model, + type: response.type, + }, + usage: { + ...getBasicUsageStats(input.messages, message), + inputTokens: response.usage.input_tokens, + outputTokens: response.usage.output_tokens, + }, + }; +} + +function toGenkitFinishReason( + reason: 'end_turn' | 'max_tokens' | 'stop_sequence' | 'tool_use' | null +): ModelResponseData['finishReason'] { + switch (reason) { + case 'end_turn': + return 'stop'; + case 'max_tokens': + return 'length'; + case 'stop_sequence': + return 'stop'; + case 'tool_use': + return 'stop'; + case null: + return 'unknown'; + default: + return 'other'; + } +} + +function toAnthropicToolRequest(tool: Record): ToolUseBlock { + if (!tool.name) { + throw new Error('Tool name is required'); + } + // Validate the tool name, Anthropic only supports letters, numbers, and underscores. + // https://docs.anthropic.com/en/docs/build-with-claude/tool-use#specifying-tools + if (!/^[a-zA-Z0-9_-]{1,64}$/.test(tool.name)) { + throw new Error( + `Tool name ${tool.name} contains invalid characters. + Only letters, numbers, and underscores are allowed, + and the name must be between 1 and 64 characters long.` + ); + } + const declaration: ToolUseBlock = { + type: 'tool_use', + id: tool.ref, + name: tool.name, + input: tool.input, + }; + return declaration; +} + +function toAnthropicToolResponse(part: Part): ToolResultBlockParam { + if (!part.toolResponse?.ref) { + throw new Error('Tool response reference is required'); + } + + if (!part.toolResponse.output) { + throw new Error('Tool response output is required'); + } + + return { + type: 'tool_result', + tool_use_id: part.toolResponse.ref, + content: JSON.stringify(part.toolResponse.output), + }; +} diff --git a/js/plugins/vertexai/src/modelgarden/v2/index.ts b/js/plugins/vertexai/src/modelgarden/v2/index.ts new file mode 100644 index 0000000000..53b4d99a9d --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/v2/index.ts @@ -0,0 +1,126 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GenkitError, ModelReference, z } from 'genkit'; +import { genkitPluginV2, type GenkitPluginV2 } from 'genkit/plugin'; +import { ActionType } from 'genkit/registry'; +import { getDerivedParams } from '../../common/index.js'; +import * as anthropic from './anthropic.js'; +import * as llama from './llama.js'; +import * as mistral from './mistral.js'; +import type { PluginOptions } from './types.js'; + +export type { PluginOptions }; + +async function initializer(pluginOptions?: PluginOptions) { + const clientOptions = await getDerivedParams(pluginOptions); + return [ + ...anthropic.listKnownModels(clientOptions, pluginOptions), + ...mistral.listKnownModels(clientOptions, pluginOptions), + ...llama.listKnownModels(clientOptions, pluginOptions), + ]; +} + +async function resolver( + actionType: ActionType, + actionName: string, + pluginOptions?: PluginOptions +) { + const clientOptions = await getDerivedParams(pluginOptions); + switch (actionType) { + case 'model': + if (anthropic.isAnthropicModelName(actionName)) { + return anthropic.defineModel(actionName, clientOptions, pluginOptions); + } else if (mistral.isMistralModelName(actionName)) { + return mistral.defineModel(actionName, clientOptions, pluginOptions); + } else if (llama.isLlamaModelName(actionName)) { + return llama.defineModel(actionName, clientOptions, pluginOptions); + } + break; + } + return undefined; +} + +async function listActions(options?: PluginOptions) { + try { + const clientOptions = await getDerivedParams(options); + return [ + ...anthropic.listActions(clientOptions), + ...mistral.listActions(clientOptions), + ...llama.listActions(clientOptions), + ]; + } catch (e: unknown) { + return []; + } +} + +/** + * Add Google Cloud Vertex AI Model Garden to Genkit. + */ +export function vertexModelGardenPlugin( + options: PluginOptions +): GenkitPluginV2 { + let listActionsCache; + return genkitPluginV2({ + name: 'vertex-model-garden', + init: async () => await initializer(options), + resolve: async (actionType: ActionType, actionName: string) => + await resolver(actionType, actionName, options), + list: async () => { + if (listActionsCache) return listActionsCache; + listActionsCache = await listActions(options); + return listActionsCache; + }, + }); +} + +export type VertexModelGardenPlugin = { + (pluginOptions?: PluginOptions): GenkitPluginV2; + model( + name: anthropic.KnownModels | (anthropic.AnthropicModelName & {}), + config?: anthropic.AnthropicConfig + ): ModelReference; + model( + name: mistral.KnownModels | (mistral.MistralModelName & {}), + config: mistral.MistralConfig + ): ModelReference; + model( + name: llama.KnownModels | (llama.LlamaModelName & {}), + config: llama.LlamaConfig + ): ModelReference; + model(name: string, config?: any): ModelReference; +}; + +export const vertexModelGarden = + vertexModelGardenPlugin as VertexModelGardenPlugin; +(vertexModelGarden as any).model = ( + name: string, + config?: any +): ModelReference => { + if (anthropic.isAnthropicModelName(name)) { + return anthropic.model(name, config); + } + if (mistral.isMistralModelName(name)) { + return mistral.model(name, config); + } + if (llama.isLlamaModelName(name)) { + return llama.model(name, config); + } + throw new GenkitError({ + status: 'INVALID_ARGUMENT', + message: `model '${name}' is not a recognized model name`, + }); +}; diff --git a/js/plugins/vertexai/src/modelgarden/v2/llama.ts b/js/plugins/vertexai/src/modelgarden/v2/llama.ts new file mode 100644 index 0000000000..3893266f45 --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/v2/llama.ts @@ -0,0 +1,167 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ActionMetadata, GenkitError, z, type ModelReference } from 'genkit'; +import { + ModelInfo, + modelRef, + type GenerateRequest, + type ModelAction, +} from 'genkit/model'; +import type { GoogleAuth } from 'google-auth-library'; +import OpenAI from 'openai'; +import { getGenkitClientHeader } from '../../common/index.js'; +import { + OpenAIConfigSchema, + defineOpenaiCompatibleModel, +} from './openai_compatibility.js'; +import { PluginOptions } from './types.js'; +import { checkModelName } from './utils.js'; + +export const LlamaConfigSchema = OpenAIConfigSchema.extend({ + location: z.string().optional(), +}).passthrough(); +export type LlamaConfigSchemaType = typeof LlamaConfigSchema; +export type LlamaConfig = z.infer; + +type ConfigSchemaType = LlamaConfigSchemaType; + +function commonRef( + name: string, + info?: ModelInfo, + configSchema: ConfigSchemaType = LlamaConfigSchema +): ModelReference { + return modelRef({ + name: `vertex-model-garden/${name}`, + configSchema, + info: info ?? { + supports: { + multiturn: true, + tools: true, + media: true, + systemRole: true, + output: ['text', 'json'], + }, + }, + }); +} + +export const GENERIC_MODEL = commonRef('llama'); + +export const KNOWN_MODELS = { + 'meta/llama-4-maverick-17b-128e-instruct-maas': commonRef( + 'meta/llama-4-maverick-17b-128e-instruct-maas' + ), + 'meta/llama-4-scout-17b-16e-instruct-maas': commonRef( + 'meta/llama-4-scout-17b-16e-instruct-maas' + ), + 'meta/llama-3.3-70b-instruct-maas': commonRef( + 'meta/llama-3.3-70b-instruct-maas' + ), + 'meta/llama-3.2-90b-vision-instruct-maas': commonRef( + 'meta/llama-3.2-90b-vision-instruct-maas' + ), + 'meta/llama-3.1-405b-instruct-maas': commonRef( + 'meta/llama-3.1-405b-instruct-maas' + ), + 'meta/llama-3.1-70b-instruct-maas': commonRef( + 'meta/llama-3.1-70b-instruct-maas' + ), + 'meta/llama-3.1-8b-instruct-maas': commonRef( + 'meta/llama-3.1-8b-instruct-maas' + ), +}; +export type KnownModels = keyof typeof KNOWN_MODELS; +export type LlamaModelName = `claude-${string}`; +export function isLlamaModelName(value?: string): value is LlamaModelName { + return !!value?.startsWith('meta/llama-'); +} + +export function model( + version: string, + options: LlamaConfig = {} +): ModelReference { + const name = checkModelName(version); + + return modelRef({ + name: `vertex-model-garden/${name}`, + config: options, + configSchema: LlamaConfigSchema, + info: { + ...GENERIC_MODEL.info, + }, + }); +} + +export interface ClientOptions { + location: string; // e.g. 'us-central1' or 'us-east5' + projectId: string; + authClient: GoogleAuth; + baseUrlTemplate?: string; +} + +export function listActions(clientOptions: ClientOptions): ActionMetadata[] { + // TODO: figure out where to get the list of models. + return []; +} + +export function listKnownModels( + clientOptions: ClientOptions, + pluginOptions?: PluginOptions +) { + return Object.keys(KNOWN_MODELS).map((name) => + defineModel(name, clientOptions, pluginOptions) + ); +} + +export function defineModel( + name: string, + clientOptions: ClientOptions, + pluginOptions?: PluginOptions +): ModelAction { + const ref = model(name); + const clientFactory = async ( + request: GenerateRequest + ): Promise => { + const options = await resolveOptions(clientOptions, request.config); + return new OpenAI(options); + }; + return defineOpenaiCompatibleModel(ref, clientFactory); +} + +async function resolveOptions( + clientOptions: ClientOptions, + requestConfig?: LlamaConfig +) { + const baseUrlTemplate = + clientOptions.baseUrlTemplate ?? + 'https://{location}-aiplatform.googleapis.com/v1/projects/{projectId}/locations/{location}/endpoints/openapi'; + const location = requestConfig?.location || clientOptions.location; + const baseURL = baseUrlTemplate + .replace(/{location}/g, location) + .replace(/{projectId}/g, clientOptions.projectId); + const apiKey = await clientOptions.authClient.getAccessToken(); + if (!apiKey) { + throw new GenkitError({ + status: 'PERMISSION_DENIED', + message: 'Unable to get accessToken', + }); + } + const defaultHeaders = { + 'X-Goog-Api-Client': getGenkitClientHeader(), + }; + return { baseURL, apiKey, defaultHeaders }; +} diff --git a/js/plugins/vertexai/src/modelgarden/v2/mistral.ts b/js/plugins/vertexai/src/modelgarden/v2/mistral.ts new file mode 100644 index 0000000000..cbc6cbcec7 --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/v2/mistral.ts @@ -0,0 +1,501 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MistralGoogleCloud } from '@mistralai/mistralai-gcp'; +import { + ChatCompletionChoiceFinishReason, + ToolTypes, + type AssistantMessage, + type ChatCompletionRequest, + type ChatCompletionResponse, + type CompletionChunk, + type FunctionCall, + type Tool as MistralTool, + type SystemMessage, + type ToolCall, + type ToolMessage, + type UserMessage, +} from '@mistralai/mistralai-gcp/models/components'; +import { + ActionMetadata, + GenerationCommonConfigSchema, + z, + type GenerateRequest, + type MessageData, + type ModelReference, + type ModelResponseData, + type Part, + type Role, + type ToolRequestPart, +} from 'genkit'; +import { + GenerationCommonConfigDescriptions, + ModelInfo, + modelRef, + type ModelAction, +} from 'genkit/model'; +import { model as pluginModel } from 'genkit/plugin'; +import { getGenkitClientHeader } from '../../common/index.js'; +import { PluginOptions } from './types.js'; +import { checkModelName } from './utils.js'; + +/** + * See https://docs.mistral.ai/api/#tag/chat/operation/chat_completion_v1_chat_completions_post + */ +export const MistralConfigSchema = GenerationCommonConfigSchema.extend({ + // TODO: Update this with all the parameters in + // https://docs.mistral.ai/api/#tag/chat/operation/chat_completion_v1_chat_completions_post. + location: z.string().optional(), + topP: z + .number() + .describe( + GenerationCommonConfigDescriptions.topP + ' The default value is 1.' + ) + .optional(), +}).passthrough(); +export type MistralConfigSchemaType = typeof MistralConfigSchema; +export type MistralConfig = z.infer; + +// This contains all the config schema types +type ConfigSchemaType = MistralConfigSchemaType; + +function commonRef( + name: string, + info?: ModelInfo, + configSchema: ConfigSchemaType = MistralConfigSchema +): ModelReference { + return modelRef({ + name: `vertex-model-garden/${name}`, + configSchema, + info: info ?? { + supports: { + multiturn: true, + media: false, + tools: true, + systemRole: true, + output: ['text'], + }, + }, + }); +} + +export const GENERIC_MODEL = commonRef('mistral'); + +export const KNOWN_MODELS = { + 'mistral-large-2411': commonRef('mistral-large-2411'), + 'mistral-ocr-2505': commonRef('mistral-ocr-2505'), + 'mistral-small-2503': commonRef('mistral-small-2503'), + 'codestral-2501': commonRef('codestral-2501'), +}; +export type KnownModels = keyof typeof KNOWN_MODELS; +export type MistralModelName = `${string}tral-${string}`; +export function isMistralModelName(value?: string): value is MistralModelName { + return !!value?.includes('tral-'); +} + +export function model( + version: string, + options: MistralConfig = {} +): ModelReference { + const name = checkModelName(version); + + return modelRef({ + name: `vertex-model-garden/${name}`, + config: options, + configSchema: MistralConfigSchema, + info: { + ...GENERIC_MODEL.info, + }, + }); +} + +export function listActions(clientOptions: ClientOptions): ActionMetadata[] { + // TODO: figure out where to get a list of models maybe vertex list? + return []; +} + +interface ClientOptions { + location: string; + projectId: string; +} + +export function listKnownModels( + clientOptions: ClientOptions, + pluginOptions?: PluginOptions +) { + return Object.keys(KNOWN_MODELS).map((name) => + defineModel(name, clientOptions, pluginOptions) + ); +} + +export function defineModel( + name: string, + clientOptions: ClientOptions, + pluginOptions?: PluginOptions +): ModelAction { + const ref = model(name); + const getClient = createClientFactory(clientOptions.projectId); + + return pluginModel( + { + name: ref.name, + ...ref.info, + configSchema: ref.configSchema, + }, + async (request, { streamingRequested, sendChunk }) => { + const client = getClient( + request.config?.location || clientOptions.location + ); + const modelVersion = checkModelName(ref.name); + const mistralRequest = toMistralRequest(modelVersion, request); + const mistralOptions = { + fetchOptions: { + headers: { + 'X-Goog-Api-Client': getGenkitClientHeader(), + }, + }, + }; + if (!streamingRequested) { + // Non-streaming + const response = await client.chat.complete( + mistralRequest, + mistralOptions + ); + return fromMistralResponse(request, response); + } else { + // Streaming + const stream = await client.chat.stream(mistralRequest, mistralOptions); + for await (const event of stream) { + const parts = fromMistralCompletionChunk(event.data); + if (parts.length > 0) { + sendChunk({ + content: parts, + }); + } + } + // Get the complete response after streaming + const completeResponse = await client.chat.complete( + mistralRequest, + mistralOptions + ); + return fromMistralResponse(request, completeResponse); + } + } + ); +} + +function createClientFactory(projectId: string) { + const clients: Record = {}; + + return (region: string): MistralGoogleCloud => { + if (!region) { + throw new Error('Region is required to create Mistral client'); + } + + try { + if (!clients[region]) { + clients[region] = new MistralGoogleCloud({ + region, + projectId, + }); + } + return clients[region]; + } catch (error) { + throw new Error( + `Failed to create/retrieve Mistral client for region ${region}: ${error}` + ); + } + }; +} + +type MistralRole = 'assistant' | 'user' | 'tool' | 'system'; + +function toMistralRole(role: Role): MistralRole { + switch (role) { + case 'model': + return 'assistant'; + case 'user': + return 'user'; + case 'tool': + return 'tool'; + case 'system': + return 'system'; + default: + throw new Error(`Unknwon role ${role}`); + } +} +function toMistralToolRequest(toolRequest: Record): FunctionCall { + if (!toolRequest.name) { + throw new Error('Tool name is required'); + } + + return { + name: toolRequest.name, + // Mistral expects arguments as either a string or object + arguments: + typeof toolRequest.input === 'string' + ? toolRequest.input + : JSON.stringify(toolRequest.input), + }; +} + +export function toMistralRequest( + model: string, + input: GenerateRequest +): ChatCompletionRequest { + const messages = input.messages.map((msg) => { + // Handle regular text messages + if (msg.content.every((part) => part.text)) { + const content = msg.content.map((part) => part.text || '').join(''); + return { + role: toMistralRole(msg.role), + content, + }; + } + + // Handle assistant's tool/function calls + const toolRequest = msg.content.find((part) => part.toolRequest); + if (toolRequest?.toolRequest) { + const functionCall = toMistralToolRequest(toolRequest.toolRequest); + return { + role: 'assistant' as const, + content: null, + toolCalls: [ + { + id: toolRequest.toolRequest.ref, + type: ToolTypes.Function, + function: { + name: functionCall.name, + arguments: functionCall.arguments, + }, + }, + ], + }; + } + + // Handle tool responses + const toolResponse = msg.content.find((part) => part.toolResponse); + if (toolResponse?.toolResponse) { + return { + role: 'tool' as const, + name: toolResponse.toolResponse.name, + content: JSON.stringify(toolResponse.toolResponse.output), + toolCallId: toolResponse.toolResponse.ref, // This must match the id from tool_calls + }; + } + + return { + role: toMistralRole(msg.role), + content: msg.content.map((part) => part.text || '').join(''), + }; + }); + + validateToolSequence(messages); // This line exists but might not be running? + + const request: ChatCompletionRequest = { + model, + messages, + maxTokens: input.config?.maxOutputTokens ?? 1024, + temperature: input.config?.temperature ?? 0.7, + ...(input.config?.topP && { topP: input.config.topP }), + ...(input.config?.stopSequences && { stop: input.config.stopSequences }), + ...(input.tools && { + tools: input.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: tool.inputSchema || {}, + }, + })) as MistralTool[], + }), + }; + + return request; +} +// Helper to convert Mistral AssistantMessage content into Genkit parts +function fromMistralTextPart(content: string): Part { + return { + text: content, + }; +} + +// Helper to convert Mistral ToolCall into Genkit parts +function fromMistralToolCall(toolCall: ToolCall): ToolRequestPart { + if (!toolCall.function) { + throw new Error('Tool call must include a function definition'); + } + + return { + toolRequest: { + ref: toolCall.id, + name: toolCall.function.name, + input: + typeof toolCall.function.arguments === 'string' + ? JSON.parse(toolCall.function.arguments) + : toolCall.function.arguments, + }, + }; +} + +// Converts Mistral AssistantMessage content into Genkit parts +function fromMistralMessage(message: AssistantMessage): Part[] { + const parts: Part[] = []; + + // Handle textual content + if (typeof message.content === 'string') { + parts.push(fromMistralTextPart(message.content)); + } else if (Array.isArray(message.content)) { + // If content is an array of ContentChunk, handle each chunk + message.content.forEach((chunk) => { + if (chunk.type === 'text') { + parts.push(fromMistralTextPart(chunk.text)); + } + // Add support for other ContentChunk types here if needed + }); + } + + // Handle tool calls if present + if (message.toolCalls) { + message.toolCalls.forEach((toolCall) => { + parts.push(fromMistralToolCall(toolCall)); + }); + } + + return parts; +} + +// Maps Mistral finish reasons to Genkit finish reasons +export function fromMistralFinishReason( + reason: ChatCompletionChoiceFinishReason | undefined +): 'length' | 'unknown' | 'stop' | 'blocked' | 'other' { + switch (reason) { + case ChatCompletionChoiceFinishReason.Stop: + return 'stop'; + case ChatCompletionChoiceFinishReason.Length: + case ChatCompletionChoiceFinishReason.ModelLength: + return 'length'; + case ChatCompletionChoiceFinishReason.Error: + return 'other'; // Map generic errors to "other" + case ChatCompletionChoiceFinishReason.ToolCalls: + return 'stop'; // Assuming tool calls signify a "stop" in processing + default: + return 'other'; // For undefined or unmapped reasons + } +} + +// Converts a Mistral response to a Genkit response +export function fromMistralResponse( + _input: GenerateRequest, + response: ChatCompletionResponse +): ModelResponseData { + const firstChoice = response.choices?.[0]; + // Convert content from Mistral response to Genkit parts + const contentParts: Part[] = firstChoice?.message + ? fromMistralMessage(firstChoice.message) + : []; + + const message: MessageData = { + role: 'model', + content: contentParts, + }; + + return { + message, + finishReason: fromMistralFinishReason(firstChoice?.finishReason), + usage: { + inputTokens: response.usage.promptTokens, + outputTokens: response.usage.completionTokens, + }, + custom: { + id: response.id, + model: response.model, + created: response.created, + }, + raw: response, // Include the raw response for debugging or additional context + }; +} + +type MistralMessage = + | AssistantMessage + | ToolMessage + | SystemMessage + | UserMessage; + +// Helper function to validate tool calls and responses match +function validateToolSequence(messages: MistralMessage[]) { + const toolCalls = ( + messages.filter((m) => { + return m.role === 'assistant' && m.toolCalls; + }) as AssistantMessage[] + ).reduce((acc: ToolCall[], m) => { + if (m.toolCalls) { + return [...acc, ...m.toolCalls]; + } + return acc; + }, []); + + const toolResponses = messages.filter( + (m) => m.role === 'tool' + ) as ToolMessage[]; + + if (toolCalls.length !== toolResponses.length) { + throw new Error( + `Mismatch between tool calls (${toolCalls.length}) and responses (${toolResponses.length})` + ); + } + + toolResponses.forEach((response) => { + const matchingCall = toolCalls.find( + (call) => call.id === response.toolCallId + ); + if (!matchingCall) { + throw new Error( + `Tool response with ID ${response.toolCallId} has no matching call` + ); + } + }); +} + +export function fromMistralCompletionChunk(chunk: CompletionChunk): Part[] { + if (!chunk.choices?.[0]?.delta) return []; + + const delta = chunk.choices[0].delta; + const parts: Part[] = []; + + if (typeof delta.content === 'string') { + parts.push({ text: delta.content }); + } + + if (delta.toolCalls) { + delta.toolCalls.forEach((toolCall) => { + if (!toolCall.function) return; + + parts.push({ + toolRequest: { + ref: toolCall.id, + name: toolCall.function.name, + input: + typeof toolCall.function.arguments === 'string' + ? JSON.parse(toolCall.function.arguments) + : toolCall.function.arguments, + }, + }); + }); + } + + return parts; +} diff --git a/js/plugins/vertexai/src/modelgarden/v2/openai_compatibility.ts b/js/plugins/vertexai/src/modelgarden/v2/openai_compatibility.ts new file mode 100644 index 0000000000..0b68279652 --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/v2/openai_compatibility.ts @@ -0,0 +1,419 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Message, z } from 'genkit'; +import { + GenerationCommonConfigSchema, + type CandidateData, + type GenerateRequest, + type GenerateResponseData, + type MessageData, + type ModelAction, + type ModelReference, + type Part, + type Role, + type ToolDefinition, + type ToolRequestPart, +} from 'genkit/model'; +import { model as pluginModel } from 'genkit/plugin'; +import type OpenAI from 'openai'; +import type { + ChatCompletion, + ChatCompletionChunk, + ChatCompletionContentPart, + ChatCompletionCreateParamsNonStreaming, + ChatCompletionMessageParam, + ChatCompletionMessageToolCall, + ChatCompletionRole, + ChatCompletionTool, + CompletionChoice, +} from 'openai/resources/index.mjs'; +import { checkModelName } from './utils'; + +/** + * See https://platform.openai.com/docs/api-reference/chat/create. + */ +export const OpenAIConfigSchema = GenerationCommonConfigSchema.extend({ + // TODO: topK is not supported and some of the other common config options + // have different names in the above doc. Eg: max_completion_tokens. + // Update to use the parameters in above doc. + frequencyPenalty: z + .number() + .min(-2) + .max(2) + .describe( + 'Positive values penalize new tokens based on their ' + + "existing frequency in the text so far, decreasing the model's " + + 'likelihood to repeat the same line verbatim.' + ) + .optional(), + logitBias: z + .record( + z.string().describe('Token string.'), + z.number().min(-100).max(100).describe('Associated bias value.') + ) + .describe( + 'Controls the likelihood of specified tokens appearing ' + + 'in the generated output. Map of tokens to an associated bias ' + + 'value from -100 (which will in most cases block that token ' + + 'from being generated) to 100 (exclusive selection of the ' + + 'token which makes it more likely to be generated). Moderate ' + + 'values like -1 and 1 will change the probability of a token ' + + 'being selected to a lesser degree.' + ) + .optional(), + logProbs: z + .boolean() + .describe( + 'Whether to return log probabilities of the output tokens or not.' + ) + .optional(), + presencePenalty: z + .number() + .min(-2) + .max(2) + .describe( + 'Positive values penalize new tokens based on whether ' + + "they appear in the text so far, increasing the model's " + + 'likelihood to talk about new topics.' + ) + .optional(), + seed: z + .number() + .int() + .describe( + 'If specified, the system will make a best effort to sample ' + + 'deterministically, such that repeated requests with the same seed ' + + 'and parameters should return the same result. Determinism is not ' + + 'guaranteed, and you should refer to the system_fingerprint response ' + + 'parameter to monitor changes in the backend.' + ) + .optional(), + topLogProbs: z + .number() + .int() + .min(0) + .max(20) + .describe( + 'An integer specifying the number of most likely tokens to ' + + 'return at each token position, each with an associated log ' + + 'probability. logprobs must be set to true if this parameter is used.' + ) + .optional(), + user: z + .string() + .describe( + 'A unique identifier representing your end-user to monitor and detect abuse.' + ) + .optional(), +}).passthrough(); + +export function toOpenAIRole(role: Role): ChatCompletionRole { + switch (role) { + case 'user': + return 'user'; + case 'model': + return 'assistant'; + case 'system': + return 'system'; + case 'tool': + return 'tool'; + default: + throw new Error(`role ${role} doesn't map to an OpenAI role.`); + } +} + +function toOpenAiTool(tool: ToolDefinition): ChatCompletionTool { + return { + type: 'function', + function: { + name: tool.name, + parameters: tool.inputSchema || undefined, + }, + }; +} + +export function toOpenAiTextAndMedia(part: Part): ChatCompletionContentPart { + if (part.text) { + return { + type: 'text', + text: part.text, + }; + } else if (part.media) { + return { + type: 'image_url', + image_url: { + url: part.media.url, + }, + }; + } + throw Error( + `Unsupported genkit part fields encountered for current message role: ${JSON.stringify(part)}.` + ); +} + +export function toOpenAiMessages( + messages: MessageData[] +): ChatCompletionMessageParam[] { + const openAiMsgs: ChatCompletionMessageParam[] = []; + for (const message of messages) { + const msg = new Message(message); + const role = toOpenAIRole(message.role); + switch (role) { + case 'user': + openAiMsgs.push({ + role: role, + content: msg.content.map((part) => toOpenAiTextAndMedia(part)), + }); + break; + case 'system': + openAiMsgs.push({ + role: role, + content: msg.text, + }); + break; + case 'assistant': { + const toolCalls: ChatCompletionMessageToolCall[] = msg.content + .filter( + ( + part + ): part is Part & { + toolRequest: NonNullable; + } => Boolean(part.toolRequest) + ) + .map((part) => ({ + id: part.toolRequest.ref ?? '', + type: 'function', + function: { + name: part.toolRequest.name, + arguments: JSON.stringify(part.toolRequest.input), + }, + })); + if (toolCalls.length > 0) { + openAiMsgs.push({ + role: role, + tool_calls: toolCalls, + }); + } else { + openAiMsgs.push({ + role: role, + content: msg.text, + }); + } + break; + } + case 'tool': { + const toolResponseParts = msg.toolResponseParts(); + toolResponseParts.map((part) => { + openAiMsgs.push({ + role: role, + tool_call_id: part.toolResponse.ref ?? '', + content: + typeof part.toolResponse.output === 'string' + ? part.toolResponse.output + : JSON.stringify(part.toolResponse.output), + }); + }); + break; + } + } + } + return openAiMsgs; +} + +const finishReasonMap: Record< + CompletionChoice['finish_reason'] | 'tool_calls', + CandidateData['finishReason'] +> = { + length: 'length', + stop: 'stop', + tool_calls: 'stop', + content_filter: 'blocked', +}; + +export function fromOpenAiToolCall( + toolCall: + | ChatCompletionMessageToolCall + | ChatCompletionChunk.Choice.Delta.ToolCall +): ToolRequestPart { + if (!toolCall.function) { + throw Error( + `Unexpected openAI chunk choice. tool_calls was provided but one or more tool_calls is missing.` + ); + } + const f = toolCall.function; + return { + toolRequest: { + name: f.name!, + ref: toolCall.id, + input: f.arguments ? JSON.parse(f.arguments) : f.arguments, + }, + }; +} + +export function fromOpenAiChoice( + choice: ChatCompletion.Choice, + jsonMode = false +): CandidateData { + const toolRequestParts = choice.message.tool_calls?.map(fromOpenAiToolCall); + return { + index: choice.index, + finishReason: finishReasonMap[choice.finish_reason] || 'other', + message: { + role: 'model', + content: toolRequestParts + ? // Note: Not sure why I have to cast here exactly. + // Otherwise it thinks toolRequest must be 'undefined' if provided + (toolRequestParts as ToolRequestPart[]) + : [ + jsonMode + ? { data: JSON.parse(choice.message.content!) } + : { text: choice.message.content! }, + ], + }, + custom: {}, + }; +} + +export function fromOpenAiChunkChoice( + choice: ChatCompletionChunk.Choice, + jsonMode = false +): CandidateData { + const toolRequestParts = choice.delta.tool_calls?.map(fromOpenAiToolCall); + return { + index: choice.index, + finishReason: choice.finish_reason + ? finishReasonMap[choice.finish_reason] || 'other' + : 'unknown', + message: { + role: 'model', + content: toolRequestParts + ? (toolRequestParts as ToolRequestPart[]) + : [ + jsonMode + ? { data: JSON.parse(choice.delta.content!) } + : { text: choice.delta.content! }, + ], + }, + custom: {}, + }; +} + +export function toRequestBody( + model: ModelReference, + request: GenerateRequest +) { + const openAiMessages = toOpenAiMessages(request.messages); + const mappedModelName = checkModelName(model.name); + const body = { + model: mappedModelName, + messages: openAiMessages, + temperature: request.config?.temperature, + max_tokens: request.config?.maxOutputTokens, + top_p: request.config?.topP, + stop: request.config?.stopSequences, + frequency_penalty: request.config?.frequencyPenalty, + logit_bias: request.config?.logitBias, + logprobs: request.config?.logProbs, + presence_penalty: request.config?.presencePenalty, + seed: request.config?.seed, + top_logprobs: request.config?.topLogProbs, + user: request.config?.user, + tools: request.tools?.map(toOpenAiTool), + n: request.candidates, + } as ChatCompletionCreateParamsNonStreaming; + const response_format = request.output?.format; + if (response_format) { + if ( + response_format === 'json' && + model.info?.supports?.output?.includes('json') + ) { + body.response_format = { + type: 'json_object', + }; + } else if ( + response_format === 'text' && + model.info?.supports?.output?.includes('text') + ) { + // this is default format, don't need to set it + // body.response_format = { + // type: 'text', + // }; + } else { + throw new Error(`${response_format} format is not supported currently`); + } + } + for (const key in body) { + if (!body[key] || (Array.isArray(body[key]) && !body[key].length)) + delete body[key]; + } + return body; +} + +export function defineOpenaiCompatibleModel< + C extends typeof OpenAIConfigSchema, +>( + model: ModelReference, + clientFactory: (request: GenerateRequest) => Promise +): ModelAction { + const modelId = checkModelName(model.name); + if (!modelId) throw new Error(`Unsupported model: ${modelId}`); + + return pluginModel( + { + name: model.name, + ...model.info, + configSchema: model.configSchema, + }, + async ( + request: GenerateRequest, + { streamingRequested, sendChunk } + ): Promise => { + let response: ChatCompletion; + const client = await clientFactory(request); + const body = toRequestBody(model, request); + if (streamingRequested) { + const stream = client.beta.chat.completions.stream({ + ...body, + stream: true, + }); + for await (const chunk of stream) { + chunk.choices?.forEach((chunk) => { + const c = fromOpenAiChunkChoice(chunk); + sendChunk({ + index: c.index, + content: c.message.content, + }); + }); + } + response = await stream.finalChatCompletion(); + } else { + response = await client.chat.completions.create(body); + } + return { + candidates: response.choices.map((c) => + fromOpenAiChoice(c, request.output?.format === 'json') + ), + usage: { + inputTokens: response.usage?.prompt_tokens, + outputTokens: response.usage?.completion_tokens, + totalTokens: response.usage?.total_tokens, + }, + custom: response, + }; + } + ); +} diff --git a/js/plugins/vertexai/src/modelgarden/types.ts b/js/plugins/vertexai/src/modelgarden/v2/types.ts similarity index 90% rename from js/plugins/vertexai/src/modelgarden/types.ts rename to js/plugins/vertexai/src/modelgarden/v2/types.ts index 562ed0d52c..04b2f4e2d8 100644 --- a/js/plugins/vertexai/src/modelgarden/types.ts +++ b/js/plugins/vertexai/src/modelgarden/v2/types.ts @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ import type { ModelReference } from 'genkit'; -import type { CommonPluginOptions } from '../common/types.js'; +import type { CommonPluginOptions } from '../../common/types.js'; /** * Evaluation metric config. Use `metricSpec` to define the behavior of the metric. @@ -26,7 +26,7 @@ import type { CommonPluginOptions } from '../common/types.js'; /** Options specific to Model Garden configuration */ export interface ModelGardenOptions { - models: ModelReference[]; + models?: ModelReference[]; openAiBaseUrlTemplate?: string; } diff --git a/js/plugins/vertexai/src/modelgarden/v2/utils.ts b/js/plugins/vertexai/src/modelgarden/v2/utils.ts new file mode 100644 index 0000000000..944e4b5102 --- /dev/null +++ b/js/plugins/vertexai/src/modelgarden/v2/utils.ts @@ -0,0 +1,49 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GenkitError } from 'genkit'; + +/** + * Gets the model name without certain prefixes.. + * e.g. for "models/googleai/gemini-2.5-pro" it returns just 'gemini-2.5-pro' + * @param name A string containing the model string with possible prefixes + * @returns the model string stripped of certain prefixes + */ +export function modelName(name?: string): string | undefined { + if (!name) return name; + + // Remove any of these prefixes: + const prefixesToRemove = + /background-model\/|model\/|models\/|embedders\/|vertex-model-garden\/|vertexai\//g; + return name.replace(prefixesToRemove, ''); +} + +/** + * Gets the suffix of a model string. + * Throws if the string is empty. + * @param name A string containing the model string + * @returns the model string stripped of prefixes and guaranteed not empty. + */ +export function checkModelName(name?: string): string { + const version = modelName(name); + if (!version) { + throw new GenkitError({ + status: 'INVALID_ARGUMENT', + message: 'Model name is required.', + }); + } + return version; +} diff --git a/js/plugins/vertexai/tests/modelgarden/anthropic_test.ts b/js/plugins/vertexai/tests/modelgarden/legacy/anthropic_test.ts similarity index 99% rename from js/plugins/vertexai/tests/modelgarden/anthropic_test.ts rename to js/plugins/vertexai/tests/modelgarden/legacy/anthropic_test.ts index 2f56a91ebb..e7ff683cdc 100644 --- a/js/plugins/vertexai/tests/modelgarden/anthropic_test.ts +++ b/js/plugins/vertexai/tests/modelgarden/legacy/anthropic_test.ts @@ -25,7 +25,7 @@ import { fromAnthropicResponse, toAnthropicRequest, type AnthropicConfigSchema, -} from '../../src/modelgarden/anthropic'; +} from '../../../src/modelgarden/legacy/anthropic'; const MODEL_ID = 'modelid'; diff --git a/js/plugins/vertexai/tests/modelgarden/mistral_test.ts b/js/plugins/vertexai/tests/modelgarden/legacy/mistral_test.ts similarity index 99% rename from js/plugins/vertexai/tests/modelgarden/mistral_test.ts rename to js/plugins/vertexai/tests/modelgarden/legacy/mistral_test.ts index dc93e95a5f..68db284b66 100644 --- a/js/plugins/vertexai/tests/modelgarden/mistral_test.ts +++ b/js/plugins/vertexai/tests/modelgarden/legacy/mistral_test.ts @@ -27,7 +27,7 @@ import { fromMistralResponse, toMistralRequest, type MistralConfigSchema, -} from '../../src/modelgarden/mistral'; +} from '../../../src/modelgarden/legacy/mistral'; const MODEL_ID = 'mistral-large-2411'; diff --git a/js/plugins/vertexai/tests/modelgarden/v2/anthropic_test.ts b/js/plugins/vertexai/tests/modelgarden/v2/anthropic_test.ts new file mode 100644 index 0000000000..f3c398ff7a --- /dev/null +++ b/js/plugins/vertexai/tests/modelgarden/v2/anthropic_test.ts @@ -0,0 +1,313 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + Message, + MessageCreateParamsBase, +} from '@anthropic-ai/sdk/resources/messages.mjs'; +import * as assert from 'assert'; +import type { GenerateRequest, GenerateResponseData } from 'genkit'; +import { describe, it } from 'node:test'; +import { + fromAnthropicResponse, + toAnthropicRequest, + type AnthropicConfigSchema, +} from '../../../src/modelgarden/v2/anthropic'; + +const MODEL_ID = 'modelid'; + +describe('toAnthropicRequest', () => { + const testCases: { + should: string; + input: GenerateRequest; + expectedOutput: MessageCreateParamsBase; + }[] = [ + { + should: 'should transform genkit message (text content) correctly', + input: { + messages: [ + { + role: 'user', + content: [{ text: 'Tell a joke about dogs.' }], + }, + ], + }, + expectedOutput: { + max_tokens: 4096, + model: MODEL_ID, + messages: [ + { + role: 'user', + content: [{ type: 'text', text: 'Tell a joke about dogs.' }], + }, + ], + }, + }, + { + should: 'should transform system message', + input: { + messages: [ + { + role: 'system', + content: [{ text: 'Talk like a pirate.' }], + }, + { + role: 'user', + content: [{ text: 'Tell a joke about dogs.' }], + }, + ], + }, + expectedOutput: { + max_tokens: 4096, + model: MODEL_ID, + system: 'Talk like a pirate.', + messages: [ + { + role: 'user', + content: [{ type: 'text', text: 'Tell a joke about dogs.' }], + }, + ], + }, + }, + { + should: + 'should transform genkit message (inline base64 image content) correctly', + input: { + messages: [ + { + role: 'user', + content: [ + { text: 'describe the following image:' }, + { + media: { + contentType: 'image/jpeg', + url: 'data:image/jpeg;base64,/9j/4QDeRXhpZgAASUkqAAgAAAAGABIBAwABAAAAAQAAABoBBQABAAAAVgAAABsBBQABAAAAXgAAACgBAwABAAAAAgAAABMCAwABAAAAAQAAAGmHBAABAAAAZgAAAAAAAABIAAAAAQAAAEgAAAABAAAABwAAkAcABAAAADAyMTABkQcABAAAAAECAwCGkgcAFgAAAMAAAAAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAQAAQAAAMgAAAADoAQAAQAAAMgAAAAAAAAAQVNDSUkAAABQaWNzdW0gSUQ6IDY4N//bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/CABEIAMgAyAMBIgACEQEDEQH/xAAbAAEAAgMBAQAAAAAAAAAAAAAAAQIDBAUGB//EABgBAQEBAQEAAAAAAAAAAAAAAAABAgME/9oADAMBAAIQAxAAAAH3ZOsiYEgAmIkWEiEiEkiRYSICBVSQSRIBEhQAUAEAARMAJWYmpRBZWYmYkBQAUAAEARIgJEsViidRMKmYmW98M5uVEzQAAAAIABoa3zTLZ9M2Pltl+pvmWU+kvn+xHt7eMzHrcnlMy+mam2AAAAgEBPj9/Y+XWuTb6U1xLbOWNO29EupO1Ea85IOp6/ldXeQoAAEAgJq+G9/pteA6WjoR0ev5v1Rv8Xv8jGuTERF/W07G4yGoAAACCAE1Zz6a6/z33XKXgVv0MXzfd5+1VvY4O/E2i24AACCAgkqqlAiKzXNybOmc/j+i4eNYfQ7G/Ldjy6zdUWioupKWipbRCyYgTCKlAxzjnWcnK6PJl2c2v0+W74djUrPOO28WmguoW6sF4qLREWWVgsrBZRWvNZ1iedbyWN+u6nzfoc9++1PO82X206mx343UF4rBdQWVgtEKmIglAKiZx2TT8j6bl8uvA2e1Obj1d+M69Hm4fa78rRVrN4oLTQXisF4rBaIhLKCygrIcPhnm72znHpagdD0h6uFZOvOAoJECgRBIAC//xAApEAABBAECBQUBAAMAAAAAAAABAAIDBBEFEhATITBAFBUgIjFBIzJw/9oACAEBAAEFAv8AgGVnxyfkD4RPYz4EtuGFC9VK58LkHNPz6jv9XHCwgENyEsgQszhC7ZCGoWUNSmQ1ORN1NQztnb3MFjgh9ljjjjjjpufU9zU60T2D06hjjlfJS2jlxoRBclckrkPXJlXLkC2PVSt6dnckG1X6XpnwzPghgvSOjaesLQUYQWysAPDLlSquiH73Wu3NcxqtVn0pgVuUO2KNlyuVdG12UOppUuUP3vO+jn4exzG2opG8qWHAdNPtDLW58UpLR1VKlykUOnfP+MztOX6QySUMdG/242me1yRPazElGgYiSgMeAQHBw6tK1CBskNfUDCJLQmNOnylnDQPCkZvaFP8AatHWbIq1SGHg0dfDyv8AYRxBqAwEPzwcr+KM/YL+IfnazwysrK3cf4mHBBWVn5Z71yYwVotRlklGqSKLVXlTX5I17jMXQSGSPwrsPPqw0Jo5hp9lR0rLTPWnkMenWN8MZji8K7YfXA1eRe7yBe7yr3aZe6zqpfltTLPDKys8MrKysrKzwz8NRdlmxudgWxi5bMmNhVBobZWeOTxzxyshZWeP/8QAIBEAAgICAgIDAAAAAAAAAAAAAAEREgIwECEDIDFAUP/aAAgBAwEBPwH2j8VaYKlSpUh/ab1ob14j4nShbMe3HGShwPVgmnZFRp5OTJRqw7RUqeT50//EAB8RAAICAwACAwAAAAAAAAAAAAABAhEDEjAQIRNAUP/aAAgBAgEBPwH8uuNmxsbGxa+0lxq/DEuNCRMXsrnIXOfpWxog7Vi5ZGmtWOaItRVEXa5ZHQ8jPlZidrj/AP/EADkQAAEDAQMKAgYLAQAAAAAAAAEAAhEDEiExEBMiMjNAQVFhkTCSQnFygYKhBBQgIzRSYHCiscHh/9oACAEBAAY/Av3IipUa09SvxFPutrTPxLFvfdiXGScT9i4lXVH+ZbZ/dbUrWHlWDOyvY1X0uxVpnvHilpEEYjIB4UcIv8X6zZdIueGrGsOy0X1CfZ/6ryR7lth5Stqz5rXZ3WLfMFq/MLZv7LZv8qGg6/oiTrux6eLaxb6QVtl9F2HRE0jDnGLXJVKdV1oWbieeXBHLEnus5UJtnhyXTxZRpPE0n4ItxY7AnjkhA1HBvrUZzuFIwOSEKlQaf9KOHjWx8QXQp30etrDj/qfTmbBhF5ExgEXvkkqCIRou1Th0OTOVBp8ByUDcI9A4dEHNMOGBVSqK7peZiLk6k8Q4FQCAcRKmoWwPy3oLOVYngOS67jBwKsOx4HnkzuFRmBVlzVotMlZyoPvOXLJJx3KOPDI8LSbKtNZpc8k9t1I/RLngSrGiFgsELPEIK1EX7m5l89Fask3LZnstkhZp8FpNPZRBG5h7ZuV7Hd1dTefWVsj5lsv5K5ndydSc0Na1s4zuebmMCrn8FtFrke5azo9lTaePhVRzCYdwI3H/xAAqEAACAQIFAgYDAQEAAAAAAAABEQAhMRBBUWFxIDCBkaGx0fBAweFw8f/aAAgBAQABPyH/AABYmsY/GZQW6WZqfhZQgGfpgoK59ASAvvmW9lU8p/IEFt4JBjBuAgZy8oq2LxY5wegoC+8y0OZDUwAoBdsAg9gVLGOClq9fM6+QDArm5CZkk5t+cXJfyguXHdJ0ikTI4HQ4tHJSrAIARbABFqICTRWspEEWT4QwILj7qoPUADxMzB5l8QLBTqLgb90L/uDK8X4IXb6OJtPIPYvHzSjQydjfuKHyJ/QPiVgqEgyqYND3D4d1oqwJTY1lPiXUDfpKozeYFYc/qVAfN2Agi6bhwZxkJPiVUAlAa2j0JgehnYAowIChGz5mv/rujU4IOR0h4D0AHI6fE2uKUDQ7iFIVAG0IFWaGziGMxHEPqL3BEuxEWMMCsKk0gBFpH3eGtOXeEo2PMGvMAwGoMpqYAz7BClRIpkbRZyQyxJjJZpqYMVWtWHQebgVFGdIMRWfp3hM5ntAAADvgde+JpxAXGuDDHOCoxCqGPvMKOOpZ4wVNixFvGGIFVXeDIqPIikrrCc45nvuGInQMMGGFdB84BWGgMw7GE7K0vElitdmIAJrfW8VnoIatxi+p9bjgMsFS0MogkFoPeetxow6lRfUwmXTKzTBxx4OOPqcccccdJdANwhRwa9oTChx9hxxx9DGAmriA4TJM0JhUdDjjjjjjwOJF1i6xNcTQ+MPMcMUHjADSM6xnWM6xnWNGdYzrGdY444444THiFgVqyhwCCbQk2UMJo+yc2FVAJMAMRjBUCGxjjjlca9Pj0uOEHHMCR/jOLQD58rSjgxuiCWhQENQImoSS9+prE4sRjBxiNZQHaAJpVAN60ELs66YSKRsuNtESChbv+YfovtAWYHcNYScDnCHAfUjhHgcHOBKCRubRlQkb1zymTc60y8o0020CqNNZoqBa4BBxGAIWAdnnie8Z2nBGYzDDGkYGUTOGBDecI45//9oADAMBAAIAAwAAABBhJAAuAIJODC8BIgAcABSwEABb+gg6kAFzww0BA2VeEAbzzzzwEABOYQKszTzziED9uo7LMuvzzwED+T//AGn90884hEzOM7iPE4whC/Nwm7969CeLeTKcADfPOfnOGS0kgwBv7elCCCmwMoMyt33g4A4AAV9xefhCjhABAgffA//EAB4RAAMBAAIDAQEAAAAAAAAAAAABERAgMSEwQVFh/9oACAEDAQE/EOExoQntfK4sWwnB6hvT5dka2JXF6tuvv0LKs+HGlxMs7If0fhyu0bqGGG8UvPyOgxspUVFWUpcVoTtia6PIDs/UpfATWloGN55UpcWAlf1iQWR6f//EAB4RAAMAAgMBAQEAAAAAAAAAAAABERAhIDAxQVFh/9oACAECAQE/EOVLyfa8b60qJJYY+loPBJI3dDw0RjTN/glF0N6glRppvEvehmjRfjP4PpyhMMQdE2WleEIQhCYhB6NmeSEIQhCExsZM8F6O9ESeEREIQnJo8GMei4ovR1UNwU8SGlxJDKvk8//EACkQAQACAgECBgIDAQEBAAAAAAEAESExQVFhEHGBkaGxMMEg0fBA4fH/2gAIAQEAAT8QgeBrwD+FQjqUypUqV+EPAPE14mv+KvEL8T874PjUPxX+F8TwNf8AOeF0QXT8QcsFQTSP5alfwZUrwvWebrLB7w8QNL7zh93/ABNyIbrntNORj0Srw7gj+orYCRwW68OtRscpAAmn8L/ItqZnPRb8TQuvX90IM50/vmAMOZ+4hHMeq5cTqdGomtZ6xMjxAOUPsTDjWN8QRdepuGcev5bg65ESndlwVa83Vf3EIQ+kpbAHsQhRp1f2mYH7GvmYMvTMfMrP9g6QzzL+tBtv90fuUK8mR+2Wsdcv6kmSZKDT9z8rD3EUhhGK0Ab4DmKsGMKov99oAitaTVvrNhm4xaRqrIZ4KJb5S14UTa3cAA16wxfS5QzYd7ltEaw733uLi4fXSqfdh+RSXYhQMZRyYLOEdETRCNlFfrajFcxmHzIo06cq3ssyxVvoH9pXWPQ/c0PkKfYS2VjRk+4iAyHZCeRtHhh2DHxcvaWuZspIBis0ZNw8gA1oBwee8P5P8sTbK6GsOUNnJ5EMt1BWZE9Hh9IJCnQJ7FmrLmKZs83UUDza1XaWwcmGGCbdLiGwZtqf+TEwDHVfLp3jqL7Gn3iwU6aDkjSopZHPFdZgYAqne/yoZ4YuDq6+X5TxI9wLa7kCorqGy9t7XCV0gYcxMRuj+jqRDR+Bq39xWGVkekOSkD7JBb66lQZYwL6wXQjaL7g6hf8AyJm5p3XVRwKqu3R++7iZuphnPaABQUH5XTEqo0cez5PIlYV9a42I+ySn1CgUj5RpP0zWa79VXp5RKxMS0sWcgCpLUE2ct/nylDG6RYvvHcAWzWWJ0GqTv2iICRKBdvSFUrtMhf8ALeNR7LqWtOvnNJB/K/G5fitQloYzvD8u7jo45i8qflo8icjyQU3EiNeeaH1qLWWv2sUeSrX0zBDTo1vWgZpHZLBNstA1aCiEvZ0DKTRXdQiXlbvs694tRHafb2gFLUym1/EsGpcuLAQnZKHki9hbHz9nPvKOeY0NUecAeoZvt5XD30UGAfPp7w4vSK2RxQVcNuDeBO5Tf10cqntvg2L0IbBd/Y4DtLlyhLPG4udzzRaikvwXPhlzBERJsUdlpiqC5mYZavZL8GggkHkn1AoOpUTTS69JRQOtTiVYCYHL5suWlplLlpad0slxZcW4giJWGEsTZ5QPul55AHzKgLTtKgFDZbZkmLKeC+8vvL7y+8vvLlhKeBjKXFzLIdaA6kG29L6mU3mZV9v3HVAs01H4IFwpXLlstO6Uj4NesqckRKx6h7ynhG/SI8IGNsB6sbk8vqDZKG+j6ZYbl5931KXuPqyo2neTvJ3kU5Z3k7yPWS/Vi+rHHcXUvL9YjzFHMt6y5YAARJvmbuwC9V6xSp4Cg4sgxadq/qLhYuUVX6QgoE2Gr8pYcZ5ikLipaWixb1guoo3LUje7mes9Y7h1uevhzmUdQtBBRWcVOPmYMyO58oOrkcWEwC6HJ9R9ptZ7+IMdQ4pedbilJAdquJXNxSoAtxQlzJFx5ziZYmo1d0yqWSyIvmWTNi0XqlYhZRWQeo/UeBDBMwXmUXrU/wAMQylncn6hxS8BA8q+0K6DjZ8S9gCXeoA0V7QTxHKn4le8v1MStQFR733iThlb1G+os7xFZqNCiFvFRVQ7os8mCnVlSrQTZksBafGFMbFabqDpQDANvh5t+8b4RwWOb5sls01QtHzqcUUqD69YMXdONZAm6W5YvLfvADSssaHrG3cC8S3al3BKOBlhf3lCEYUYX0gnSgVwhj1ktsCf/9k=', + }, + }, + ], + }, + ], + }, + expectedOutput: { + max_tokens: 4096, + model: MODEL_ID, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'describe the following image:' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: '/9j/4QDeRXhpZgAASUkqAAgAAAAGABIBAwABAAAAAQAAABoBBQABAAAAVgAAABsBBQABAAAAXgAAACgBAwABAAAAAgAAABMCAwABAAAAAQAAAGmHBAABAAAAZgAAAAAAAABIAAAAAQAAAEgAAAABAAAABwAAkAcABAAAADAyMTABkQcABAAAAAECAwCGkgcAFgAAAMAAAAAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAQAAQAAAMgAAAADoAQAAQAAAMgAAAAAAAAAQVNDSUkAAABQaWNzdW0gSUQ6IDY4N//bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/CABEIAMgAyAMBIgACEQEDEQH/xAAbAAEAAgMBAQAAAAAAAAAAAAAAAQIDBAUGB//EABgBAQEBAQEAAAAAAAAAAAAAAAABAgME/9oADAMBAAIQAxAAAAH3ZOsiYEgAmIkWEiEiEkiRYSICBVSQSRIBEhQAUAEAARMAJWYmpRBZWYmYkBQAUAAEARIgJEsViidRMKmYmW98M5uVEzQAAAAIABoa3zTLZ9M2Pltl+pvmWU+kvn+xHt7eMzHrcnlMy+mam2AAAAgEBPj9/Y+XWuTb6U1xLbOWNO29EupO1Ea85IOp6/ldXeQoAAEAgJq+G9/pteA6WjoR0ev5v1Rv8Xv8jGuTERF/W07G4yGoAAACCAE1Zz6a6/z33XKXgVv0MXzfd5+1VvY4O/E2i24AACCAgkqqlAiKzXNybOmc/j+i4eNYfQ7G/Ldjy6zdUWioupKWipbRCyYgTCKlAxzjnWcnK6PJl2c2v0+W74djUrPOO28WmguoW6sF4qLREWWVgsrBZRWvNZ1iedbyWN+u6nzfoc9++1PO82X206mx343UF4rBdQWVgtEKmIglAKiZx2TT8j6bl8uvA2e1Obj1d+M69Hm4fa78rRVrN4oLTQXisF4rBaIhLKCygrIcPhnm72znHpagdD0h6uFZOvOAoJECgRBIAC//xAApEAABBAECBQUBAAMAAAAAAAABAAIDBBEFEhATITBAFBUgIjFBIzJw/9oACAEBAAEFAv8AgGVnxyfkD4RPYz4EtuGFC9VK58LkHNPz6jv9XHCwgENyEsgQszhC7ZCGoWUNSmQ1ORN1NQztnb3MFjgh9ljjjjjjpufU9zU60T2D06hjjlfJS2jlxoRBclckrkPXJlXLkC2PVSt6dnckG1X6XpnwzPghgvSOjaesLQUYQWysAPDLlSquiH73Wu3NcxqtVn0pgVuUO2KNlyuVdG12UOppUuUP3vO+jn4exzG2opG8qWHAdNPtDLW58UpLR1VKlykUOnfP+MztOX6QySUMdG/242me1yRPazElGgYiSgMeAQHBw6tK1CBskNfUDCJLQmNOnylnDQPCkZvaFP8AatHWbIq1SGHg0dfDyv8AYRxBqAwEPzwcr+KM/YL+IfnazwysrK3cf4mHBBWVn5Z71yYwVotRlklGqSKLVXlTX5I17jMXQSGSPwrsPPqw0Jo5hp9lR0rLTPWnkMenWN8MZji8K7YfXA1eRe7yBe7yr3aZe6zqpfltTLPDKys8MrKysrKzwz8NRdlmxudgWxi5bMmNhVBobZWeOTxzxyshZWeP/8QAIBEAAgICAgIDAAAAAAAAAAAAAAEREgIwECEDIDFAUP/aAAgBAwEBPwH2j8VaYKlSpUh/ab1ob14j4nShbMe3HGShwPVgmnZFRp5OTJRqw7RUqeT50//EAB8RAAICAwACAwAAAAAAAAAAAAABAhEDEjAQIRNAUP/aAAgBAgEBPwH8uuNmxsbGxa+0lxq/DEuNCRMXsrnIXOfpWxog7Vi5ZGmtWOaItRVEXa5ZHQ8jPlZidrj/AP/EADkQAAEDAQMKAgYLAQAAAAAAAAEAAhEDEiExEBMiMjNAQVFhkTCSQnFygYKhBBQgIzRSYHCiscHh/9oACAEBAAY/Av3IipUa09SvxFPutrTPxLFvfdiXGScT9i4lXVH+ZbZ/dbUrWHlWDOyvY1X0uxVpnvHilpEEYjIB4UcIv8X6zZdIueGrGsOy0X1CfZ/6ryR7lth5Stqz5rXZ3WLfMFq/MLZv7LZv8qGg6/oiTrux6eLaxb6QVtl9F2HRE0jDnGLXJVKdV1oWbieeXBHLEnus5UJtnhyXTxZRpPE0n4ItxY7AnjkhA1HBvrUZzuFIwOSEKlQaf9KOHjWx8QXQp30etrDj/qfTmbBhF5ExgEXvkkqCIRou1Th0OTOVBp8ByUDcI9A4dEHNMOGBVSqK7peZiLk6k8Q4FQCAcRKmoWwPy3oLOVYngOS67jBwKsOx4HnkzuFRmBVlzVotMlZyoPvOXLJJx3KOPDI8LSbKtNZpc8k9t1I/RLngSrGiFgsELPEIK1EX7m5l89Fask3LZnstkhZp8FpNPZRBG5h7ZuV7Hd1dTefWVsj5lsv5K5ndydSc0Na1s4zuebmMCrn8FtFrke5azo9lTaePhVRzCYdwI3H/xAAqEAACAQIFAgYDAQEAAAAAAAABEQAhMRBBUWFxIDCBkaGx0fBAweFw8f/aAAgBAQABPyH/AABYmsY/GZQW6WZqfhZQgGfpgoK59ASAvvmW9lU8p/IEFt4JBjBuAgZy8oq2LxY5wegoC+8y0OZDUwAoBdsAg9gVLGOClq9fM6+QDArm5CZkk5t+cXJfyguXHdJ0ikTI4HQ4tHJSrAIARbABFqICTRWspEEWT4QwILj7qoPUADxMzB5l8QLBTqLgb90L/uDK8X4IXb6OJtPIPYvHzSjQydjfuKHyJ/QPiVgqEgyqYND3D4d1oqwJTY1lPiXUDfpKozeYFYc/qVAfN2Agi6bhwZxkJPiVUAlAa2j0JgehnYAowIChGz5mv/rujU4IOR0h4D0AHI6fE2uKUDQ7iFIVAG0IFWaGziGMxHEPqL3BEuxEWMMCsKk0gBFpH3eGtOXeEo2PMGvMAwGoMpqYAz7BClRIpkbRZyQyxJjJZpqYMVWtWHQebgVFGdIMRWfp3hM5ntAAADvgde+JpxAXGuDDHOCoxCqGPvMKOOpZ4wVNixFvGGIFVXeDIqPIikrrCc45nvuGInQMMGGFdB84BWGgMw7GE7K0vElitdmIAJrfW8VnoIatxi+p9bjgMsFS0MogkFoPeetxow6lRfUwmXTKzTBxx4OOPqcccccdJdANwhRwa9oTChx9hxxx9DGAmriA4TJM0JhUdDjjjjjjwOJF1i6xNcTQ+MPMcMUHjADSM6xnWM6xnWNGdYzrGdY444444THiFgVqyhwCCbQk2UMJo+yc2FVAJMAMRjBUCGxjjjlca9Pj0uOEHHMCR/jOLQD58rSjgxuiCWhQENQImoSS9+prE4sRjBxiNZQHaAJpVAN60ELs66YSKRsuNtESChbv+YfovtAWYHcNYScDnCHAfUjhHgcHOBKCRubRlQkb1zymTc60y8o0020CqNNZoqBa4BBxGAIWAdnnie8Z2nBGYzDDGkYGUTOGBDecI45//9oADAMBAAIAAwAAABBhJAAuAIJODC8BIgAcABSwEABb+gg6kAFzww0BA2VeEAbzzzzwEABOYQKszTzziED9uo7LMuvzzwED+T//AGn90884hEzOM7iPE4whC/Nwm7969CeLeTKcADfPOfnOGS0kgwBv7elCCCmwMoMyt33g4A4AAV9xefhCjhABAgffA//EAB4RAAMBAAIDAQEAAAAAAAAAAAABERAgMSEwQVFh/9oACAEDAQE/EOExoQntfK4sWwnB6hvT5dka2JXF6tuvv0LKs+HGlxMs7If0fhyu0bqGGG8UvPyOgxspUVFWUpcVoTtia6PIDs/UpfATWloGN55UpcWAlf1iQWR6f//EAB4RAAMAAgMBAQEAAAAAAAAAAAABERAhIDAxQVFh/9oACAECAQE/EOVLyfa8b60qJJYY+loPBJI3dDw0RjTN/glF0N6glRppvEvehmjRfjP4PpyhMMQdE2WleEIQhCYhB6NmeSEIQhCExsZM8F6O9ESeEREIQnJo8GMei4ovR1UNwU8SGlxJDKvk8//EACkQAQACAgECBgIDAQEBAAAAAAEAESExQVFhEHGBkaGxMMEg0fBA4fH/2gAIAQEAAT8QgeBrwD+FQjqUypUqV+EPAPE14mv+KvEL8T874PjUPxX+F8TwNf8AOeF0QXT8QcsFQTSP5alfwZUrwvWebrLB7w8QNL7zh93/ABNyIbrntNORj0Srw7gj+orYCRwW68OtRscpAAmn8L/ItqZnPRb8TQuvX90IM50/vmAMOZ+4hHMeq5cTqdGomtZ6xMjxAOUPsTDjWN8QRdepuGcev5bg65ESndlwVa83Vf3EIQ+kpbAHsQhRp1f2mYH7GvmYMvTMfMrP9g6QzzL+tBtv90fuUK8mR+2Wsdcv6kmSZKDT9z8rD3EUhhGK0Ab4DmKsGMKov99oAitaTVvrNhm4xaRqrIZ4KJb5S14UTa3cAA16wxfS5QzYd7ltEaw733uLi4fXSqfdh+RSXYhQMZRyYLOEdETRCNlFfrajFcxmHzIo06cq3ssyxVvoH9pXWPQ/c0PkKfYS2VjRk+4iAyHZCeRtHhh2DHxcvaWuZspIBis0ZNw8gA1oBwee8P5P8sTbK6GsOUNnJ5EMt1BWZE9Hh9IJCnQJ7FmrLmKZs83UUDza1XaWwcmGGCbdLiGwZtqf+TEwDHVfLp3jqL7Gn3iwU6aDkjSopZHPFdZgYAqne/yoZ4YuDq6+X5TxI9wLa7kCorqGy9t7XCV0gYcxMRuj+jqRDR+Bq39xWGVkekOSkD7JBb66lQZYwL6wXQjaL7g6hf8AyJm5p3XVRwKqu3R++7iZuphnPaABQUH5XTEqo0cez5PIlYV9a42I+ySn1CgUj5RpP0zWa79VXp5RKxMS0sWcgCpLUE2ct/nylDG6RYvvHcAWzWWJ0GqTv2iICRKBdvSFUrtMhf8ALeNR7LqWtOvnNJB/K/G5fitQloYzvD8u7jo45i8qflo8icjyQU3EiNeeaH1qLWWv2sUeSrX0zBDTo1vWgZpHZLBNstA1aCiEvZ0DKTRXdQiXlbvs694tRHafb2gFLUym1/EsGpcuLAQnZKHki9hbHz9nPvKOeY0NUecAeoZvt5XD30UGAfPp7w4vSK2RxQVcNuDeBO5Tf10cqntvg2L0IbBd/Y4DtLlyhLPG4udzzRaikvwXPhlzBERJsUdlpiqC5mYZavZL8GggkHkn1AoOpUTTS69JRQOtTiVYCYHL5suWlplLlpad0slxZcW4giJWGEsTZ5QPul55AHzKgLTtKgFDZbZkmLKeC+8vvL7y+8vvLlhKeBjKXFzLIdaA6kG29L6mU3mZV9v3HVAs01H4IFwpXLlstO6Uj4NesqckRKx6h7ynhG/SI8IGNsB6sbk8vqDZKG+j6ZYbl5931KXuPqyo2neTvJ3kU5Z3k7yPWS/Vi+rHHcXUvL9YjzFHMt6y5YAARJvmbuwC9V6xSp4Cg4sgxadq/qLhYuUVX6QgoE2Gr8pYcZ5ikLipaWixb1guoo3LUje7mes9Y7h1uevhzmUdQtBBRWcVOPmYMyO58oOrkcWEwC6HJ9R9ptZ7+IMdQ4pedbilJAdquJXNxSoAtxQlzJFx5ziZYmo1d0yqWSyIvmWTNi0XqlYhZRWQeo/UeBDBMwXmUXrU/wAMQylncn6hxS8BA8q+0K6DjZ8S9gCXeoA0V7QTxHKn4le8v1MStQFR733iThlb1G+os7xFZqNCiFvFRVQ7os8mCnVlSrQTZksBafGFMbFabqDpQDANvh5t+8b4RwWOb5sls01QtHzqcUUqD69YMXdONZAm6W5YvLfvADSssaHrG3cC8S3al3BKOBlhf3lCEYUYX0gnSgVwhj1ktsCf/9k=', + }, + }, + ], + }, + ], + }, + }, + ]; + for (const test of testCases) { + it(test.should, () => { + assert.deepEqual( + toAnthropicRequest(MODEL_ID, test.input), + test.expectedOutput + ); + }); + } +}); + +describe('fromAnthropicResponse', () => { + const testCases: { + should: string; + input: GenerateRequest; + response: Message; + expectedOutput: GenerateResponseData; + }[] = [ + { + should: 'should transform genkit message (text content) correctly', + input: { + messages: [ + { + role: 'user', + content: [{ text: 'Tell a joke about dogs.' }], + }, + ], + }, + response: { + id: 'abcd1234', + model: MODEL_ID, + role: 'assistant', + stop_reason: 'end_turn', + usage: { + input_tokens: 123, + output_tokens: 234, + }, + stop_sequence: null, + type: 'message', + content: [ + { + type: 'text', + text: 'part 1', + }, + { + type: 'text', + text: 'part 2', + }, + ], + }, + expectedOutput: { + custom: { + id: 'abcd1234', + model: MODEL_ID, + type: 'message', + }, + finishReason: 'stop', + message: { + role: 'model', + content: [ + { + text: 'part 1', + }, + { + text: 'part 2', + }, + ], + }, + usage: { + inputAudioFiles: 0, + inputCharacters: 23, + inputImages: 0, + inputTokens: 123, + inputVideos: 0, + outputAudioFiles: 0, + outputCharacters: 12, + outputImages: 0, + outputTokens: 234, + outputVideos: 0, + }, + }, + }, + { + should: 'should transform genkit tool call correctly', + input: { + messages: [ + { + role: 'user', + content: [{ text: "What's the weather like today?" }], + }, + ], + tools: [ + { + name: 'get_weather', + description: 'Get the weather for a location.', + inputSchema: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + ], + }, + response: { + id: 'abcd1234', + model: MODEL_ID, + role: 'assistant', + type: 'message', + stop_reason: 'tool_use', + stop_sequence: null, + usage: { + input_tokens: 123, + output_tokens: 234, + }, + content: [ + { + id: 'toolu_get_weather', + name: 'get_weather', + type: 'tool_use', + input: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + ], + }, + expectedOutput: { + custom: { + id: 'abcd1234', + model: MODEL_ID, + type: 'message', + }, + finishReason: 'stop', + message: { + role: 'model', + content: [ + { + toolRequest: { + name: 'get_weather', + ref: 'toolu_get_weather', + input: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + }, + ], + }, + usage: { + inputAudioFiles: 0, + inputCharacters: 30, + inputImages: 0, + inputTokens: 123, + inputVideos: 0, + outputAudioFiles: 0, + outputCharacters: 0, + outputImages: 0, + outputTokens: 234, + outputVideos: 0, + }, + }, + }, + ]; + for (const test of testCases) { + it(test.should, () => { + assert.deepEqual( + fromAnthropicResponse(test.input, test.response), + test.expectedOutput + ); + }); + } +}); diff --git a/js/plugins/vertexai/tests/modelgarden/v2/index_test.ts b/js/plugins/vertexai/tests/modelgarden/v2/index_test.ts new file mode 100644 index 0000000000..ac88d2c463 --- /dev/null +++ b/js/plugins/vertexai/tests/modelgarden/v2/index_test.ts @@ -0,0 +1,100 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { GenkitError } from 'genkit'; +import { describe, it } from 'node:test'; +import { vertexModelGarden } from '../../../src/modelgarden/v2'; +import { + AnthropicConfigSchema, + isAnthropicModelName, +} from '../../../src/modelgarden/v2/anthropic'; +import { + LlamaConfigSchema, + isLlamaModelName, +} from '../../../src/modelgarden/v2/llama'; +import { + MistralConfigSchema, + isMistralModelName, +} from '../../../src/modelgarden/v2/mistral'; +import { modelName as stripPrefix } from '../../../src/modelgarden/v2/utils'; + +describe('vertexModelGarden.model helper', () => { + it('should return an Anthropic model reference', () => { + const modelName = 'claude-3-haiku@20240307'; + const model = vertexModelGarden.model(modelName); + assert.ok(isAnthropicModelName(stripPrefix(model.name))); + assert.strictEqual(model.name, `vertex-model-garden/${modelName}`); + assert.deepStrictEqual(model.configSchema, AnthropicConfigSchema); + }); + + it('should return a Mistral model reference', () => { + const modelName = 'mistral-large-2411'; + const model = vertexModelGarden.model(modelName); + assert.ok(isMistralModelName(stripPrefix(model.name))); + assert.strictEqual(model.name, `vertex-model-garden/${modelName}`); + assert.deepStrictEqual(model.configSchema, MistralConfigSchema); + }); + + it('should return a Llama model reference', () => { + const modelName = 'meta/llama-3.1-8b-instruct-maas'; + const model = vertexModelGarden.model(modelName); + assert.ok(isLlamaModelName(stripPrefix(model.name))); + assert.strictEqual(model.name, `vertex-model-garden/${modelName}`); + assert.deepStrictEqual(model.configSchema, LlamaConfigSchema); + }); + + it('should return an Anthropic model reference for a pattern-matched name', () => { + const modelName = 'claude-foo'; + const model = vertexModelGarden.model(modelName); + assert.ok(isAnthropicModelName(stripPrefix(model.name))); + assert.strictEqual(model.name, `vertex-model-garden/${modelName}`); + assert.deepStrictEqual(model.configSchema, AnthropicConfigSchema); + }); + + it('should return a Mistral model reference for a pattern-matched name', () => { + const modelName = 'mistral-foo'; + const model = vertexModelGarden.model(modelName); + assert.ok(isMistralModelName(stripPrefix(model.name))); + assert.strictEqual(model.name, `vertex-model-garden/${modelName}`); + assert.deepStrictEqual(model.configSchema, MistralConfigSchema); + }); + + it('should return a Llama model reference for a pattern-matched name', () => { + const modelName = 'meta/llama-foo'; + const model = vertexModelGarden.model(modelName); + assert.ok(isLlamaModelName(stripPrefix(model.name))); + assert.strictEqual(model.name, `vertex-model-garden/${modelName}`); + assert.deepStrictEqual(model.configSchema, LlamaConfigSchema); + }); + + it('should throw an error for an unrecognized model name', () => { + const modelName = 'unrecognized-model'; + assert.throws( + () => { + vertexModelGarden.model(modelName); + }, + (err: GenkitError) => { + assert.strictEqual(err.status, 'INVALID_ARGUMENT'); + assert.strictEqual( + err.message, + `INVALID_ARGUMENT: model '${modelName}' is not a recognized model name` + ); + return true; + } + ); + }); +}); diff --git a/js/plugins/vertexai/tests/modelgarden/v2/llama_test.ts b/js/plugins/vertexai/tests/modelgarden/v2/llama_test.ts new file mode 100644 index 0000000000..cf01f69cfa --- /dev/null +++ b/js/plugins/vertexai/tests/modelgarden/v2/llama_test.ts @@ -0,0 +1,149 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { GenerateRequest, modelRef } from 'genkit/model'; +import { describe, it } from 'node:test'; +import { LlamaConfigSchema } from '../../../src/modelgarden/v2/llama'; +import { toRequestBody } from '../../../src/modelgarden/v2/openai_compatibility'; + +const fakeModel = modelRef({ + name: 'vertex-model-garden/meta/llama-4-maverick-17b-128e-instruct-maas', + info: { + supports: { + multiturn: true, + tools: true, + media: false, + systemRole: true, + output: ['text', 'json'], + }, + }, + configSchema: LlamaConfigSchema, +}); + +describe('Llama request conversion', () => { + it('should convert a simple request', () => { + const genkitRequest: GenerateRequest = { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + }; + const openAIRequest = toRequestBody(fakeModel, genkitRequest); + assert.deepStrictEqual( + openAIRequest.model, + 'meta/llama-4-maverick-17b-128e-instruct-maas' + ); + assert.deepStrictEqual(openAIRequest.messages, [ + { role: 'user', content: [{ type: 'text', text: 'Hello' }] }, + ]); + }); + + it('should convert a request with history', () => { + const genkitRequest: GenerateRequest = { + messages: [ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'model', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: 'How are you?' }] }, + ], + }; + const openAIRequest = toRequestBody(fakeModel, genkitRequest); + assert.deepStrictEqual(openAIRequest.messages, [ + { role: 'user', content: [{ type: 'text', text: 'Hello' }] }, + { role: 'assistant', content: 'Hi there!' }, + { role: 'user', content: [{ type: 'text', text: 'How are you?' }] }, + ]); + }); + + it('should convert a request with tools', () => { + const genkitRequest: GenerateRequest = { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + tools: [ + { + name: 'my_tool', + description: 'a tool', + inputSchema: { + type: 'object', + properties: { foo: { type: 'string' } }, + }, + }, + ], + }; + const openAIRequest = toRequestBody(fakeModel, genkitRequest); + assert.deepStrictEqual(openAIRequest.tools, [ + { + type: 'function', + function: { + name: 'my_tool', + parameters: { + type: 'object', + properties: { foo: { type: 'string' } }, + }, + }, + }, + ]); + }); + + it('should convert a tool call response', () => { + const genkitRequest: GenerateRequest = { + messages: [ + { role: 'user', content: [{ text: 'Search for cats' }] }, + { + role: 'model', + content: [ + { + toolRequest: { + name: 'search', + ref: 'tool1', + input: { query: 'cats' }, + }, + }, + ], + }, + { + role: 'tool', + content: [ + { + toolResponse: { + ref: 'tool1', + name: 'search', + output: { + results: ['cat1.jpg', 'cat2.jpg'], + }, + }, + }, + ], + }, + ], + }; + const openAIRequest = toRequestBody(fakeModel, genkitRequest); + assert.deepStrictEqual(openAIRequest.messages, [ + { role: 'user', content: [{ type: 'text', text: 'Search for cats' }] }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool1', + type: 'function', + function: { name: 'search', arguments: '{"query":"cats"}' }, + }, + ], + }, + { + role: 'tool', + tool_call_id: 'tool1', + content: '{"results":["cat1.jpg","cat2.jpg"]}', + }, + ]); + }); +}); diff --git a/js/plugins/vertexai/tests/modelgarden/v2/mistral_test.ts b/js/plugins/vertexai/tests/modelgarden/v2/mistral_test.ts new file mode 100644 index 0000000000..4d8d72b15e --- /dev/null +++ b/js/plugins/vertexai/tests/modelgarden/v2/mistral_test.ts @@ -0,0 +1,688 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + ChatCompletionRequest, + ChatCompletionResponse, + CompletionChunk, +} from '@mistralai/mistralai-gcp/models/components'; +import * as assert from 'assert'; +import type { GenerateRequest, GenerateResponseData } from 'genkit'; +import { describe, it } from 'node:test'; +import { + fromMistralCompletionChunk, + fromMistralResponse, + toMistralRequest, + type MistralConfigSchema, +} from '../../../src/modelgarden/v2/mistral'; + +const MODEL_ID = 'mistral-large-2411'; + +describe('toMistralRequest', () => { + const testCases: { + should: string; + input: GenerateRequest; + expectedOutput: ChatCompletionRequest; + }[] = [ + { + should: 'should transform genkit message (text content) correctly', + input: { + messages: [ + { + role: 'user' as const, + content: [{ text: 'Tell a joke about dogs.' }], + }, + ], + }, + expectedOutput: { + model: MODEL_ID, + messages: [ + { + role: 'user', + content: 'Tell a joke about dogs.', + }, + ], + maxTokens: 1024, + temperature: 0.7, + }, + }, + { + should: 'should transform system message', + input: { + messages: [ + { + role: 'system' as const, + content: [{ text: 'Talk like a pirate.' }], + }, + { + role: 'user' as const, + content: [{ text: 'Tell a joke about dogs.' }], + }, + ], + }, + expectedOutput: { + model: MODEL_ID, + messages: [ + { + role: 'system', + content: 'Talk like a pirate.', + }, + { + role: 'user', + content: 'Tell a joke about dogs.', + }, + ], + maxTokens: 1024, + temperature: 0.7, + }, + }, + { + should: 'should transform tool request correctly', + input: { + messages: [ + { + role: 'user' as const, + content: [{ text: "What's the weather like today?" }], + }, + ], + tools: [ + { + name: 'get_weather', + description: 'Get the weather for a location.', + inputSchema: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + ], + }, + expectedOutput: { + model: MODEL_ID, + messages: [ + { + role: 'user', + content: "What's the weather like today?", + }, + ], + maxTokens: 1024, + temperature: 0.7, + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the weather for a location.', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + }, + ], + }, + }, + ]; + for (const test of testCases) { + it(test.should, () => { + assert.deepEqual( + toMistralRequest(MODEL_ID, test.input), + test.expectedOutput + ); + }); + } +}); + +describe('fromMistralResponse', () => { + const testCases: { + should: string; + input: GenerateRequest; + response: ChatCompletionResponse; + expectedOutput: GenerateResponseData; + }[] = [ + { + should: 'should transform mistral message (text content) correctly', + input: { + messages: [ + { + role: 'user' as const, + content: [{ text: 'Tell a joke about dogs.' }], + }, + ], + }, + response: { + id: 'abcd1234', + object: 'chat.completion', + model: MODEL_ID, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: + 'Why do dogs make terrible comedians? Their jokes are too ruff!', + }, + finishReason: 'stop', + }, + ], + usage: { + promptTokens: 123, + completionTokens: 234, + totalTokens: 357, + }, + }, + expectedOutput: { + message: { + role: 'model' as const, + content: [ + { + text: 'Why do dogs make terrible comedians? Their jokes are too ruff!', + }, + ], + }, + finishReason: 'stop', + usage: { + inputTokens: 123, + outputTokens: 234, + }, + custom: { + id: 'abcd1234', + model: MODEL_ID, + created: undefined, + }, + raw: { + id: 'abcd1234', + object: 'chat.completion', + model: MODEL_ID, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: + 'Why do dogs make terrible comedians? Their jokes are too ruff!', + }, + finishReason: 'stop', + }, + ], + usage: { + promptTokens: 123, + completionTokens: 234, + totalTokens: 357, + }, + }, + }, + }, + { + should: 'should transform tool calls correctly', + input: { + messages: [ + { + role: 'user' as const, + content: [{ text: "What's the weather like today?" }], + }, + ], + }, + response: { + id: 'abcd1234', + object: 'chat.completion', + model: MODEL_ID, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'call_abc123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"San Francisco, CA"}', + }, + }, + ], + }, + finishReason: 'tool_calls', + }, + ], + usage: { + promptTokens: 123, + completionTokens: 234, + totalTokens: 357, + }, + }, + expectedOutput: { + message: { + role: 'model' as const, + content: [ + { + toolRequest: { + ref: 'call_abc123', + name: 'get_weather', + input: { location: 'San Francisco, CA' }, + }, + }, + ], + }, + finishReason: 'stop', + usage: { + inputTokens: 123, + outputTokens: 234, + }, + custom: { + id: 'abcd1234', + model: MODEL_ID, + created: undefined, + }, + raw: { + id: 'abcd1234', + object: 'chat.completion', + model: MODEL_ID, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'call_abc123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"San Francisco, CA"}', + }, + }, + ], + }, + finishReason: 'tool_calls', + }, + ], + usage: { + promptTokens: 123, + completionTokens: 234, + totalTokens: 357, + }, + }, + }, + }, + ]; + for (const test of testCases) { + it(test.should, () => { + assert.deepEqual( + fromMistralResponse(test.input, test.response), + test.expectedOutput + ); + }); + } +}); + +describe('validateToolSequence', () => { + it('should handle valid tool call and response sequence', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [{ text: "What's the weather?" }], + }, + { + role: 'model' as const, + content: [ + { + toolRequest: { + ref: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ], + }, + { + role: 'tool' as const, + content: [ + { + toolResponse: { + ref: 'call_123', + name: 'get_weather', + output: { temperature: 72, condition: 'sunny' }, + }, + }, + ], + }, + ], + }; + + // Should not throw an error + assert.doesNotThrow(() => toMistralRequest(MODEL_ID, input)); + }); + + it('should throw error when tool response is missing', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [{ text: "What's the weather?" }], + }, + { + role: 'model' as const, + content: [ + { + toolRequest: { + ref: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ], + }, + ], + }; + + assert.throws( + () => toMistralRequest(MODEL_ID, input), + /Mismatch between tool calls/ + ); + }); + + it('should throw error when tool response id does not match call', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [{ text: "What's the weather?" }], + }, + { + role: 'model' as const, + content: [ + { + toolRequest: { + ref: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ], + }, + { + role: 'tool' as const, + content: [ + { + toolResponse: { + ref: 'wrong_id', + name: 'get_weather', + output: { temperature: 72, condition: 'sunny' }, + }, + }, + ], + }, + ], + }; + + assert.throws( + () => toMistralRequest(MODEL_ID, input), + /Tool response with ID wrong_id has no matching call/ + ); + }); +}); + +describe('edge cases', () => { + it('should handle empty message content', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [], + }, + ], + }; + + const result = toMistralRequest(MODEL_ID, input); + assert.equal(result.messages[0].content, ''); + }); + + it('should handle multiple text parts in content', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [{ text: 'Hello' }, { text: ' ' }, { text: 'World' }], + }, + ], + }; + + const result = toMistralRequest(MODEL_ID, input); + assert.equal(result.messages[0].content, 'Hello World'); + }); + + it('should handle custom configuration options', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [{ text: 'test' }], + }, + ], + config: { + temperature: 0.5, + maxOutputTokens: 500, + topP: 0.9, + stopSequences: ['END'], + }, + }; + + const result = toMistralRequest(MODEL_ID, input); + assert.equal(result.temperature, 0.5); + assert.equal(result.maxTokens, 500); + assert.equal(result.topP, 0.9); + assert.deepEqual(result.stop, ['END']); + }); +}); + +describe('fromMistralResponse error handling', () => { + it('should handle response with no choices', () => { + const input = { + messages: [ + { + role: 'user' as const, + content: [{ text: 'test' }], + }, + ], + }; + + const response = { + id: 'test', + object: 'chat.completion', + model: MODEL_ID, + choices: [], + usage: { + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + }, + }; + + const result = fromMistralResponse(input, response); + assert.deepEqual(result.message!.content, []); + }); +}); + +describe('fromMistralCompletionChunk', () => { + it('should handle text content chunk', () => { + const chunk = { + id: 'chunk_1', + model: MODEL_ID, + choices: [ + { + index: 0, + delta: { + role: 'assistant', + content: 'Hello world', + }, + }, + ], + }; + + const parts = fromMistralCompletionChunk( + chunk as unknown as CompletionChunk + ); + assert.deepEqual(parts, [{ text: 'Hello world' }]); + }); + + it('should handle tool call chunk', () => { + const chunk = { + id: 'chunk_1', + model: MODEL_ID, + choices: [ + { + index: 0, + delta: { + role: 'assistant', + toolCalls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"San Francisco"}', + }, + }, + ], + }, + }, + ], + }; + + const parts = fromMistralCompletionChunk( + chunk as unknown as CompletionChunk + ); + assert.deepEqual(parts, [ + { + toolRequest: { + ref: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ]); + }); + + it('should handle empty chunk', () => { + const chunk = { + id: 'chunk_1', + model: MODEL_ID, + choices: [], + }; + + const parts = fromMistralCompletionChunk(chunk); + assert.deepEqual(parts, []); + }); + + it('should handle chunk with empty delta', () => { + const chunk = { + id: 'chunk_1', + model: MODEL_ID, + choices: [ + { + index: 0, + delta: {}, + }, + ], + }; + + const parts = fromMistralCompletionChunk( + chunk as unknown as CompletionChunk + ); + assert.deepEqual(parts, []); + }); + + it('should handle chunk with invalid tool call', () => { + const chunk = { + id: 'chunk_1', + model: MODEL_ID, + choices: [ + { + index: 0, + delta: { + toolCalls: [ + { + id: 'call_123', + type: 'function', + // Missing function property + }, + ], + }, + }, + ], + }; + + const parts = fromMistralCompletionChunk( + chunk as unknown as CompletionChunk + ); + assert.deepEqual(parts, []); // Should skip invalid tool call + }); + + it('should handle mixed content chunk', () => { + const chunk = { + id: 'chunk_1', + model: MODEL_ID, + choices: [ + { + index: 0, + delta: { + content: 'Some text', + toolCalls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"San Francisco"}', + }, + }, + ], + }, + }, + ], + }; + + const parts = fromMistralCompletionChunk( + chunk as unknown as CompletionChunk + ); + assert.deepEqual(parts, [ + { text: 'Some text' }, + { + toolRequest: { + ref: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ]); + }); +}); diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index d392a9d21b..c78fddd0ac 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -181,7 +181,7 @@ importers: optionalDependencies: '@genkit-ai/firebase': specifier: ^1.16.1 - version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) doc-snippets: dependencies: @@ -1017,7 +1017,7 @@ importers: version: link:../../plugins/compat-oai '@genkit-ai/express': specifier: ^1.1.0 - version: 1.12.0(@genkit-ai/core@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit) + version: 1.12.0(@genkit-ai/core@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit) genkit: specifier: workspace:* version: link:../../genkit @@ -1641,9 +1641,9 @@ importers: testapps/model-tester: dependencies: - '@genkit-ai/googleai': + '@genkit-ai/google-genai': specifier: workspace:* - version: link:../../plugins/googleai + version: link:../../plugins/google-genai '@genkit-ai/vertexai': specifier: workspace:* version: link:../../plugins/vertexai @@ -1661,7 +1661,7 @@ importers: version: link:../../plugins/ollama genkitx-openai: specifier: ^0.10.1 - version: 0.10.1(@genkit-ai/ai@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13) + version: 0.10.1(@genkit-ai/ai@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13) devDependencies: rimraf: specifier: ^6.0.1 @@ -1866,6 +1866,9 @@ importers: '@genkit-ai/firebase': specifier: workspace:* version: link:../../plugins/firebase + '@genkit-ai/google-genai': + specifier: workspace:* + version: link:../../plugins/google-genai '@genkit-ai/vertexai': specifier: workspace:* version: link:../../plugins/vertexai @@ -2662,11 +2665,11 @@ packages: '@firebase/webchannel-wrapper@1.0.3': resolution: {integrity: sha512-2xCRM9q9FlzGZCdgDMJwc0gyUkWFtkosy7Xxr6sFgQwn+wMNIWd7xIvYNauU1r64B5L5rsGKy/n9TKJ0aAFeqQ==} - '@genkit-ai/ai@1.20.0-rc.2': - resolution: {integrity: sha512-XTTTNvkC/2w9gSMxKTXaNFc2UpafP83CtvkXbSL2GNja7Gtta7pDyweg2TUOJPzeD5aG62ztuVdBrJYmv0LFNA==} + '@genkit-ai/ai@1.20.0': + resolution: {integrity: sha512-mT8rS5Qc3pLKM4nLIRo2PWP7PgNln/4/z3hC96DxIXJ6MGE33nbEqU2X4bHvB33bKxiDgMOahTbOu0APNj8BtA==} - '@genkit-ai/core@1.20.0-rc.2': - resolution: {integrity: sha512-D2RikUJRtR62E2PhNGbF4bquYmyweMokMgAZZyr/7GRSkR5x3P2FoJRAvZMLpCwq8u2LzPC80KreUZSw0Jv6jQ==} + '@genkit-ai/core@1.20.0': + resolution: {integrity: sha512-E0ZL8p3ZXt7pBKj4GzCkLA5dBoUMULLAeFNYzoBFfjndmZhflmo3JgjkefoaCHJi8fl+1pAeF2a+1iqMY/KktQ==} '@genkit-ai/express@1.12.0': resolution: {integrity: sha512-QAxSS07dX5ovSfsUB4s90KaDnv4zg1wnoxCZCa+jBsYUyv9NvCCTsOk25xAQgGxc7xi3+MD+3AsPier5oZILIg==} @@ -5307,8 +5310,8 @@ packages: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} engines: {node: '>=14'} - genkit@1.20.0-rc.2: - resolution: {integrity: sha512-M+b1qsom9HJoUwcX3KS5R18AORnSjdcg2OCnSnx7gNrd3M69MX1HlGYYnrlIwlQb5chK/+cCtFEzOcLgjl53WA==} + genkit@1.20.0: + resolution: {integrity: sha512-PqsvZVrwHBsFaJTVtXtblSqJqj/cZYlk09ar7u7whrO2mzn+giFR+tdnnXIGJQZZnfzy7GbyvhFU1h18TLlUYQ==} genkitx-openai@0.10.1: resolution: {integrity: sha512-E9/DzyQcBUSTy81xT2pvEmdnn9Q/cKoojEt6lD/EdOeinhqE9oa59d/kuXTokCMekTrj3Rk7LtNBQIDjnyjNOA==} @@ -8510,9 +8513,9 @@ snapshots: '@firebase/webchannel-wrapper@1.0.3': {} - '@genkit-ai/ai@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/ai@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: - '@genkit-ai/core': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/core': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) '@opentelemetry/api': 1.9.0 '@types/node': 20.19.1 colorette: 2.0.20 @@ -8531,9 +8534,9 @@ snapshots: - supports-color optional: true - '@genkit-ai/ai@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': + '@genkit-ai/ai@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': dependencies: - '@genkit-ai/core': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) '@opentelemetry/api': 1.9.0 '@types/node': 20.19.1 colorette: 2.0.20 @@ -8551,7 +8554,7 @@ snapshots: - genkit - supports-color - '@genkit-ai/core@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/core@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) @@ -8573,7 +8576,7 @@ snapshots: zod: 3.25.67 zod-to-json-schema: 3.24.5(zod@3.25.67) optionalDependencies: - '@genkit-ai/firebase': 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/firebase': 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) transitivePeerDependencies: - '@google-cloud/firestore' - encoding @@ -8583,7 +8586,7 @@ snapshots: - supports-color optional: true - '@genkit-ai/core@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': + '@genkit-ai/core@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) @@ -8614,9 +8617,9 @@ snapshots: - genkit - supports-color - '@genkit-ai/express@1.12.0(@genkit-ai/core@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit)': + '@genkit-ai/express@1.12.0(@genkit-ai/core@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit)': dependencies: - '@genkit-ai/core': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) body-parser: 1.20.3 cors: 2.8.5 express: 5.1.0 @@ -8624,12 +8627,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@genkit-ai/firebase@1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/firebase@1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: - '@genkit-ai/google-cloud': 1.16.1(encoding@0.1.13)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/google-cloud': 1.16.1(encoding@0.1.13)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) '@google-cloud/firestore': 7.11.1(encoding@0.1.13) firebase-admin: 13.4.0(encoding@0.1.13) - genkit: 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) + genkit: 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) optionalDependencies: firebase: 11.9.1 transitivePeerDependencies: @@ -8650,7 +8653,7 @@ snapshots: - supports-color optional: true - '@genkit-ai/google-cloud@1.16.1(encoding@0.1.13)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/google-cloud@1.16.1(encoding@0.1.13)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: '@google-cloud/logging-winston': 6.0.1(encoding@0.1.13)(winston@3.17.0) '@google-cloud/opentelemetry-cloud-monitoring-exporter': 0.19.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) @@ -8666,7 +8669,7 @@ snapshots: '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': 0.52.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) - genkit: 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) + genkit: 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) google-auth-library: 9.15.1(encoding@0.1.13) node-fetch: 3.3.2 winston: 3.17.0 @@ -11691,10 +11694,10 @@ snapshots: - encoding - supports-color - genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): + genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): dependencies: - '@genkit-ai/ai': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) - '@genkit-ai/core': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/ai': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/core': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) uuid: 10.0.0 transitivePeerDependencies: - '@google-cloud/firestore' @@ -11704,10 +11707,10 @@ snapshots: - supports-color optional: true - genkitx-openai@0.10.1(@genkit-ai/ai@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13): + genkitx-openai@0.10.1(@genkit-ai/ai@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13): dependencies: - '@genkit-ai/ai': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) - '@genkit-ai/core': 1.20.0-rc.2(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/ai': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.20.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) openai: 4.104.0(encoding@0.1.13)(zod@3.25.67) zod: 3.25.67 transitivePeerDependencies: diff --git a/js/testapps/model-tester/package.json b/js/testapps/model-tester/package.json index 5b1d4af626..63c55be296 100644 --- a/js/testapps/model-tester/package.json +++ b/js/testapps/model-tester/package.json @@ -9,14 +9,15 @@ "build": "pnpm build:clean && pnpm compile", "build:clean": "rimraf ./lib", "build:watch": "tsc --watch", - "build-and-run": "pnpm build && node lib/index.js" + "build-and-run": "pnpm build && node lib/index.js", + "genkit:dev": "genkit start -- npx tsx --watch src/index.ts" }, "keywords": [], "author": "", "license": "ISC", "dependencies": { "genkit": "workspace:*", - "@genkit-ai/googleai": "workspace:*", + "@genkit-ai/google-genai": "workspace:*", "@genkit-ai/vertexai": "workspace:*", "@google/generative-ai": "^0.15.0", "colorette": "^2.0.20", diff --git a/js/testapps/model-tester/src/index.ts b/js/testapps/model-tester/src/index.ts index 46a1e18cf0..dd721a19d7 100644 --- a/js/testapps/model-tester/src/index.ts +++ b/js/testapps/model-tester/src/index.ts @@ -14,13 +14,8 @@ * limitations under the License. */ -import { googleAI } from '@genkit-ai/googleai'; -import { vertexAI } from '@genkit-ai/vertexai'; -import { - claude3Sonnet, - llama31, - vertexAIModelGarden, -} from '@genkit-ai/vertexai/modelgarden'; +import { googleAI, vertexAI } from '@genkit-ai/google-genai'; +import { vertexModelGarden } from '@genkit-ai/vertexai/modelgarden'; import * as clc from 'colorette'; import { genkit } from 'genkit'; import { testModels } from 'genkit/testing'; @@ -33,9 +28,8 @@ export const ai = genkit({ vertexAI({ location: 'us-central1', }), - vertexAIModelGarden({ + vertexModelGarden({ location: 'us-central1', - models: [claude3Sonnet, llama31], }), ollama({ models: [ @@ -51,12 +45,12 @@ export const ai = genkit({ }); testModels(ai.registry, [ - 'googleai/gemini-1.5-pro-latest', - 'googleai/gemini-1.5-flash-latest', - 'vertexai/gemini-1.5-pro', - 'vertexai/gemini-1.5-flash', - 'vertexai/claude-3-sonnet', - 'vertexai/llama-3.1', + 'googleai/gemini-2.5-pro', + 'googleai/gemini-2.5-flash', + 'vertexai/gemini-2.5-pro', + 'vertexai/gemini-2.5-flash', + 'vertex-model-garden/claude-sonnet-4@20250514', + 'vertex-model-garden/meta/llama-4-maverick-17b-128e-instruct-maas', 'ollama/gemma2', // 'openai/gpt-4o', // 'openai/gpt-4o-mini', diff --git a/js/testapps/vertexai-modelgarden/package.json b/js/testapps/vertexai-modelgarden/package.json index 50260e8d4d..b841c739f7 100644 --- a/js/testapps/vertexai-modelgarden/package.json +++ b/js/testapps/vertexai-modelgarden/package.json @@ -1,11 +1,10 @@ { "main": "lib/index.js", "scripts": { - "start:anthropic": "node lib/anthropic.js", - "start:mistral": "node lib/mistral.js", "build": "tsc", "build:watch": "tsc --watch", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "genkit:dev": "genkit start -- npx tsx --watch src/index.ts" }, "name": "anthropic-models", "version": "1.0.0", @@ -16,6 +15,7 @@ "dependencies": { "@genkit-ai/firebase": "workspace:*", "@genkit-ai/vertexai": "workspace:*", + "@genkit-ai/google-genai": "workspace:*", "@mistralai/mistralai-gcp": "^1.3.4", "express": "^4.21.0", "genkit": "workspace:*", diff --git a/js/testapps/vertexai-modelgarden/src/anthropic.ts b/js/testapps/vertexai-modelgarden/src/anthropic.ts deleted file mode 100644 index b14d505363..0000000000 --- a/js/testapps/vertexai-modelgarden/src/anthropic.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Import models from the Vertex AI plugin. The Vertex AI API provides access to -// several generative models. Here, we import Gemini 1.5 Flash. -import { vertexAI } from '@genkit-ai/vertexai'; -import { - claude35Sonnet, - vertexAIModelGarden, -} from '@genkit-ai/vertexai/modelgarden'; -// Import the Genkit core libraries and plugins. -import { genkit, z } from 'genkit'; - -// Import models from the Vertex AI plugin. The Vertex AI API provides access to -// several generative models. Here, we import Gemini 1.5 Flash. - -const ai = genkit({ - plugins: [ - // Load the Vertex AI plugin. You can optionally specify your project ID - // by passing in a config object; if you don't, the Vertex AI plugin uses - // the value from the GCLOUD_PROJECT environment variable. - vertexAI({ - location: 'europe-west1', - }), - vertexAIModelGarden({ - location: 'europe-west1', - models: [claude35Sonnet], - }), - ], -}); - -ai.defineTool( - { - name: 'menu-suggestion', - description: 'Generate a menu suggestion for a themed restaurant', - inputSchema: z.object({ - subject: z.string(), - }), - outputSchema: z.object({ - menuItems: z.array(z.string()), - }), - }, - async () => { - return { - menuItems: [`Appetizer: Meow Salad`], - }; - } -); - -// Define a simple flow that prompts an LLM to generate menu suggestions. -export const menuSuggestionFlow = ai.defineFlow( - { - name: 'menuSuggestionFlow', - inputSchema: z.string(), - outputSchema: z.array(z.any()), - }, - async (subject) => { - const prompt = `Suggest an item for the menu of a ${subject} themed restaurant`; - const llmResponse = await ai.generate({ - model: claude35Sonnet, - prompt: prompt, - tools: ['menu-suggestion'], - config: { - temperature: 1, - }, - returnToolRequests: true, - }); - - return llmResponse.toolRequests; - } -); diff --git a/js/testapps/vertexai-modelgarden/src/index.ts b/js/testapps/vertexai-modelgarden/src/index.ts new file mode 100644 index 0000000000..1159be7cae --- /dev/null +++ b/js/testapps/vertexai-modelgarden/src/index.ts @@ -0,0 +1,259 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { vertexAI } from '@genkit-ai/google-genai'; +import { + mistralLarge, + vertexAIModelGarden, + vertexModelGarden, +} from '@genkit-ai/vertexai/modelgarden'; +// Import the Genkit core libraries and plugins. +import { genkit, z } from 'genkit'; + +// Import models from the Vertex AI plugin. The Vertex AI API provides access to +// several generative models. Here, we import Gemini 1.5 Flash. + +const ai = genkit({ + plugins: [ + // Load the Vertex AI plugin. You can optionally specify your project ID + // by passing in a config object; if you don't, the Vertex AI plugin uses + // the value from the GCLOUD_PROJECT environment variable. + vertexAI({ + location: 'us-central1', + }), + vertexModelGarden({ + location: 'us-central1', + }), + vertexAIModelGarden({ + location: 'us-central1', + models: [mistralLarge], + }), + ], +}); + +export const anthropicModel = ai.defineFlow( + { + name: 'claude-sonnet-4 - toolCallingFlow', + inputSchema: z.string().default('Paris, France'), + outputSchema: z.string(), + streamSchema: z.any(), + }, + async (location, { sendChunk }) => { + const { response, stream } = ai.generateStream({ + model: vertexModelGarden.model('claude-sonnet-4@20250514'), + config: { + temperature: 1, + location: 'us-east5', + }, + tools: [getWeather, celsiusToFahrenheit], + prompt: `What's the weather in ${location}? Convert the temperature to Fahrenheit.`, + }); + + for await (const chunk of stream) { + sendChunk(chunk); + } + + return (await response).text; + } +); + +export const llamaModel = ai.defineFlow( + { + name: 'llama4 - basicFlow', + outputSchema: z.string(), + streamSchema: z.any(), + }, + async (location, { sendChunk }) => { + const { response, stream } = ai.generateStream({ + model: vertexModelGarden.model( + 'meta/llama-4-maverick-17b-128e-instruct-maas' + ), + config: { + temperature: 1, + location: 'us-east5', + }, + prompt: `You are a helpful assistant named Walt. Say hello`, + }); + + for await (const chunk of stream) { + sendChunk(chunk); + } + + return (await response).text; + } +); + +// Mistral Large for detailed explanations +export const mistralExplainConcept = ai.defineFlow( + { + name: 'mistral-large - explainConcept', + inputSchema: z.object({ + concept: z.string().default('concurrency'), + }), + outputSchema: z.object({ + explanation: z.string(), + examples: z.array(z.string()), + }), + }, + async ({ concept }) => { + const explanation = await ai.generate({ + model: vertexModelGarden.model('mistral-large-2411'), + prompt: `Explain ${concept} in programming. Include practical examples.`, + config: { + temperature: 0.7, + }, + output: { + schema: z.object({ + explanation: z.string(), + examples: z.array(z.string()), + }), + }, + }); + + return explanation.output || { explanation: '', examples: [] }; + } +); + +export const legacyMistralExplainConcept = ai.defineFlow( + { + name: 'legacy-mistral-large - explainConcept', + inputSchema: z.object({ + concept: z.string().default('concurrency'), + }), + outputSchema: z.object({ + explanation: z.string(), + examples: z.array(z.string()), + }), + }, + async ({ concept }) => { + const explanation = await ai.generate({ + model: mistralLarge, + prompt: `Explain ${concept} in programming. Include practical examples.`, + config: { + version: 'mistral-large-2411', + temperature: 0.7, + }, + output: { + schema: z.object({ + explanation: z.string(), + examples: z.array(z.string()), + }), + }, + }); + + return explanation.output || { explanation: '', examples: [] }; + } +); + +// Mistral small for quick validation and analysis +export const analyzeCode = ai.defineFlow( + { + name: 'mistral-small - analyzeCode', + inputSchema: z.object({ + code: z.string().default("console.log('hello world');"), + }), + outputSchema: z.string(), + }, + async ({ code }) => { + const analysis = await ai.generate({ + model: vertexModelGarden.model('mistral-small-2503'), + prompt: `Analyze this code for potential issues and suggest improvements: + ${code}`, + }); + + return analysis.text; + } +); + +// Codestral for code generation +export const generateFunction = ai.defineFlow( + { + name: 'codestral - generateFunction', + inputSchema: z.object({ + description: z.string().default('greets me and asks my favourite colour'), + }), + outputSchema: z.string(), + }, + async ({ description }) => { + const result = await ai.generate({ + model: vertexModelGarden.model('codestral-2501'), + prompt: `Create a TypeScript function that ${description}. Include error handling and types.`, + }); + + return result.text; + } +); + +// No naming collisions +export const geminiModel = ai.defineFlow( + { + name: 'gemini-2.5-flash - tool flow', + inputSchema: z.string().default('Paris, France'), + outputSchema: z.string(), + streamSchema: z.any(), + }, + async (location, { sendChunk }) => { + const { response, stream } = ai.generateStream({ + model: vertexAI.model('gemini-2.5-flash'), + config: { + temperature: 1, + }, + tools: [getWeather, celsiusToFahrenheit], + prompt: `What's the weather in ${location}? Convert the temperature to Fahrenheit.`, + }); + + for await (const chunk of stream) { + sendChunk(chunk); + } + + return (await response).text; + } +); + +const getWeather = ai.defineTool( + { + name: 'getWeather', + inputSchema: z.object({ + location: z + .string() + .describe( + 'Location for which to get the weather, ex: San-Francisco, CA' + ), + }), + description: 'used to get current weather for a location', + }, + async (input) => { + // pretend we call an actual API + return { + location: input.location, + temperature_celcius: 21.5, + conditions: 'cloudy', + }; + } +); + +const celsiusToFahrenheit = ai.defineTool( + { + name: 'celsiusToFahrenheit', + inputSchema: z.object({ + celsius: z.number().describe('Temperature in Celsius'), + }), + description: 'Converts Celsius to Fahrenheit', + }, + async ({ celsius }) => { + return (celsius * 9) / 5 + 32; + } +); diff --git a/js/testapps/vertexai-modelgarden/src/mistral.ts b/js/testapps/vertexai-modelgarden/src/mistral.ts deleted file mode 100644 index 19ce217ae0..0000000000 --- a/js/testapps/vertexai-modelgarden/src/mistral.ts +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { vertexAI } from '@genkit-ai/vertexai'; -import { - codestral, - mistralLarge, - mistralNemo, - vertexAIModelGarden, -} from '@genkit-ai/vertexai/modelgarden'; -import { genkit, z } from 'genkit'; - -const ai = genkit({ - plugins: [ - vertexAI({ - location: 'europe-west4', - }), - vertexAIModelGarden({ - location: 'europe-west4', - models: [mistralLarge, mistralNemo, codestral], - }), - ], -}); - -// Mistral Nemo for quick validation and analysis -export const analyzeCode = ai.defineFlow( - { - name: 'analyzeCode', - inputSchema: z.object({ - code: z.string(), - }), - outputSchema: z.string(), - }, - async ({ code }) => { - const analysis = await ai.generate({ - model: mistralNemo, - prompt: `Analyze this code for potential issues and suggest improvements: - ${code}`, - }); - - return analysis.text; - } -); - -// Codestral for code generation -export const generateFunction = ai.defineFlow( - { - name: 'generateFunction', - inputSchema: z.object({ - description: z.string(), - }), - outputSchema: z.string(), - }, - async ({ description }) => { - const result = await ai.generate({ - model: codestral, - prompt: `Create a TypeScript function that ${description}. Include error handling and types.`, - }); - - return result.text; - } -); - -// Mistral Large for detailed explanations -export const explainConcept = ai.defineFlow( - { - name: 'explainConcept', - inputSchema: z.object({ - concept: z.string(), - }), - outputSchema: z.object({ - explanation: z.string(), - examples: z.array(z.string()), - }), - }, - async ({ concept }) => { - const explanation = await ai.generate({ - model: mistralLarge, - prompt: `Explain ${concept} in programming. Include practical examples.`, - config: { - version: 'mistral-large-2407', - temperature: 0.7, - }, - output: { - schema: z.object({ - explanation: z.string(), - examples: z.array(z.string()), - }), - }, - }); - - return explanation.output || { explanation: '', examples: [] }; - } -);