From b4526bf4fbb640aeb160b332afb7b6742662e289 Mon Sep 17 00:00:00 2001 From: Divyanshu Soni Date: Fri, 21 Nov 2025 06:29:14 +0000 Subject: [PATCH 1/4] fix: move url utils from core to openai adapters --- core/llm/llms/Anthropic.ts | 25 ++++++--- core/llm/llms/Bedrock.ts | 10 ++-- core/llm/llms/Gemini.ts | 36 ++++++++---- core/llm/llms/Ollama.ts | 11 +++- core/util/url.ts | 8 +++ .../openai-adapters/src/apis/Anthropic.ts | 25 ++++++--- packages/openai-adapters/src/apis/Bedrock.ts | 55 ++++++++++--------- packages/openai-adapters/src/index.ts | 1 + packages/openai-adapters/src/util/url.ts | 21 +++++++ 9 files changed, 133 insertions(+), 59 deletions(-) create mode 100644 packages/openai-adapters/src/util/url.ts diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index ff3f85e51eb..4d128e91d16 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -30,6 +30,7 @@ import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; import { DEFAULT_REASONING_TOKENS } from "../constants.js"; import { BaseLLM } from "../index.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; class Anthropic extends BaseLLM { static providerName = "anthropic"; @@ -105,14 +106,22 @@ class Anthropic extends BaseLLM { }); } } else { - parts.push({ - type: "image", - source: { - type: "base64", - media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), - data: part.imageUrl.url.split(",")[1], - }, - }); + const base64Data = extractBase64FromDataUrl(part.imageUrl.url); + if (base64Data) { + parts.push({ + type: "image", + source: { + type: "base64", + media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), + data: base64Data, + }, + }); + } else { + console.warn( + "Anthropic: skipping image with invalid data URL format", + part.imageUrl.url, + ); + } } } } diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index ea47b63ae1f..a3ce15aa00c 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -25,6 +25,7 @@ import { BaseLLM } from "../index.js"; import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js"; import { getSecureID } from "../utils/getSecureID.js"; import { withLLMRetry } from "../utils/retry.js"; +import { parseDataUrl } from "../../util/url.js"; interface ModelConfig { formatPayload: (text: string) => any; @@ -545,8 +546,9 @@ class Bedrock extends BaseLLM { if (part.type === "text") { blocks.push({ text: part.text }); } else if (part.type === "imageUrl" && part.imageUrl) { - try { - const [mimeType, base64Data] = part.imageUrl.url.split(","); + const parsed = parseDataUrl(part.imageUrl.url); + if (parsed) { + const { mimeType, base64Data } = parsed; const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || @@ -568,8 +570,8 @@ class Bedrock extends BaseLLM { part, ); } - } catch (error) { - console.warn("Bedrock: failed to process image part", error, part); + } else { + console.warn("Bedrock: failed to process image part", part); } } } diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 7caf2db7d75..2c7ee68b539 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -21,6 +21,7 @@ import { GeminiToolFunctionDeclaration, convertContinueToolToGeminiFunction, } from "./gemini-types"; +import { extractBase64FromDataUrl } from "../../util/url.js"; class Gemini extends BaseLLM { static providerName = "gemini"; @@ -184,16 +185,31 @@ class Gemini extends BaseLLM { } continuePartToGeminiPart(part: MessagePart): GeminiChatContentPart { - return part.type === "text" - ? { - text: part.text, - } - : { - inlineData: { - mimeType: "image/jpeg", - data: part.imageUrl?.url.split(",")[1], - }, - }; + if (part.type === "text") { + return { + text: part.text, + }; + } + + let data = ""; + if (part.imageUrl?.url) { + const extracted = extractBase64FromDataUrl(part.imageUrl.url); + if (extracted) { + data = extracted; + } else { + console.warn( + "Gemini: skipping image with invalid data URL format", + part.imageUrl.url, + ); + } + } + + return { + inlineData: { + mimeType: "image/jpeg", + data, + }, + }; } public prepareBody( diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 0a239fce909..f4f898d2de8 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -14,6 +14,7 @@ import { import { renderChatMessage } from "../../util/messageContent.js"; import { getRemoteModelInfo } from "../../util/ollamaHelper.js"; import { BaseLLM } from "../index.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; type OllamaChatMessage = { role: ChatMessageRole; @@ -303,10 +304,16 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = part.imageUrl?.url.split(",").at(-1); + const image = part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : undefined; if (image) { images.push(image); - } + } else if (part.imageUrl?.url) { + console.warn( + "Ollama: skipping image with invalid data URL format", + part.imageUrl.url, + ); } }); if (images.length > 0) { diff --git a/core/util/url.ts b/core/util/url.ts index 83e0edcba14..81131c0191f 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -1,3 +1,8 @@ +import { + parseDataUrl as parseDataUrlFromAdapter, + extractBase64FromDataUrl as extractBase64FromDataUrlFromAdapter, +} from "@continuedev/openai-adapters"; + export function canParseUrl(url: string): boolean { if ((URL as any)?.canParse) { return (URL as any).canParse(url); @@ -9,3 +14,6 @@ export function canParseUrl(url: string): boolean { return false; } } + +export const parseDataUrl = parseDataUrlFromAdapter; +export const extractBase64FromDataUrl = extractBase64FromDataUrlFromAdapter; diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index 0a7bc573cc5..b922df3f20a 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -25,6 +25,7 @@ import { CompletionUsage, } from "openai/resources/index"; import { ChatCompletionCreateParams } from "openai/resources/index.js"; +import { extractBase64FromDataUrl } from "../util/url.js"; import { AnthropicConfig } from "../types.js"; import { chatChunk, @@ -194,14 +195,22 @@ export class AnthropicApi implements BaseLlmApi { if (part.type === "image_url") { const dataUrl = part.image_url.url; if (dataUrl?.startsWith("data:")) { - blocks.push({ - type: "image", - source: { - type: "base64", - media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: dataUrl.split(",")[1], - }, - }); + const base64Data = extractBase64FromDataUrl(dataUrl); + if (base64Data) { + blocks.push({ + type: "image", + source: { + type: "base64", + media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), + data: base64Data, + }, + }); + } else { + console.warn( + "Anthropic: skipping image with invalid data URL format", + dataUrl, + ); + } } } else { const text = part.type === "text" ? part.text : part.refusal; diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index cf5588d686d..bf05414b3cc 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -31,6 +31,7 @@ import { import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; import { fromStatic } from "@aws-sdk/token-providers"; +import { parseDataUrl } from "../util/url.js"; import { BedrockConfig } from "../types.js"; import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js"; import { safeParseArgs } from "../util/parseArgs.js"; @@ -134,35 +135,35 @@ export class BedrockApi implements BaseLlmApi { throw new Error("Unsupported part type: input_audio"); case "image_url": default: - try { - const [mimeType, base64Data] = ( - part as ChatCompletionContentPartImage - ).image_url.url.split(","); - const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; - if ( - format === ImageFormat.JPEG || - format === ImageFormat.PNG || - format === ImageFormat.WEBP || - format === ImageFormat.GIF - ) { - return { - image: { - format, - source: { - bytes: Uint8Array.from(Buffer.from(base64Data, "base64")), - }, - }, - }; - } else { - console.warn( - `Bedrock: skipping unsupported image part format: ${format}`, - ); - return { text: "[Unsupported image format]" }; - } - } catch (error) { - console.warn("Bedrock: failed to process image part", error); + const parsed = parseDataUrl( + (part as ChatCompletionContentPartImage).image_url.url, + ); + if (!parsed) { + console.warn("Bedrock: failed to process image part - invalid URL"); return { text: "[Failed to process image]" }; } + const { mimeType, base64Data } = parsed; + const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; + if ( + format === ImageFormat.JPEG || + format === ImageFormat.PNG || + format === ImageFormat.WEBP || + format === ImageFormat.GIF + ) { + return { + image: { + format, + source: { + bytes: Uint8Array.from(Buffer.from(base64Data, "base64")), + }, + }, + }; + } else { + console.warn( + `Bedrock: skipping unsupported image part format: ${format}`, + ); + return { text: "[Unsupported image format]" }; + } } } diff --git a/packages/openai-adapters/src/index.ts b/packages/openai-adapters/src/index.ts index a7ee579f9f6..09b3cbacd44 100644 --- a/packages/openai-adapters/src/index.ts +++ b/packages/openai-adapters/src/index.ts @@ -190,3 +190,4 @@ export { } from "./apis/AnthropicUtils.js"; export { isResponsesModel } from "./apis/openaiResponses.js"; +export { parseDataUrl, extractBase64FromDataUrl } from "./util/url.js"; diff --git a/packages/openai-adapters/src/util/url.ts b/packages/openai-adapters/src/util/url.ts new file mode 100644 index 00000000000..be013bb2e85 --- /dev/null +++ b/packages/openai-adapters/src/util/url.ts @@ -0,0 +1,21 @@ +export function parseDataUrl(dataUrl: string): + | { + mimeType: string; + base64Data: string; + } + | undefined { + const urlParts = dataUrl.split(","); + + if (urlParts.length < 2) { + return undefined; + } + + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); + + return { mimeType, base64Data }; +} + +export function extractBase64FromDataUrl(dataUrl: string): string | undefined { + return parseDataUrl(dataUrl)?.base64Data; +} From e0759074114e60829b8d433fa74e1f5df210c6cc Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:59:37 +0530 Subject: [PATCH 2/4] lint fix --- core/llm/llms/Ollama.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index f4f898d2de8..33712d98d80 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -314,6 +314,7 @@ class Ollama extends BaseLLM implements ModelInstaller { "Ollama: skipping image with invalid data URL format", part.imageUrl.url, ); + } } }); if (images.length > 0) { From 70dfab942f26fc525738e1c7649acaf90006348e Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Fri, 21 Nov 2025 13:09:05 +0530 Subject: [PATCH 3/4] fix run prettier --- packages/openai-adapters/src/apis/Bedrock.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index bf05414b3cc..d98ad25d2f5 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -31,10 +31,10 @@ import { import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; import { fromStatic } from "@aws-sdk/token-providers"; -import { parseDataUrl } from "../util/url.js"; import { BedrockConfig } from "../types.js"; import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js"; import { safeParseArgs } from "../util/parseArgs.js"; +import { parseDataUrl } from "../util/url.js"; import { BaseLlmApi, CreateRerankResponse, @@ -163,7 +163,7 @@ export class BedrockApi implements BaseLlmApi { `Bedrock: skipping unsupported image part format: ${format}`, ); return { text: "[Unsupported image format]" }; - } + } } } From 1c7668c3fd1954b13951e0f0068461bb05e16b7e Mon Sep 17 00:00:00 2001 From: dcs-soni <52232900+dcs-soni@users.noreply.github.com> Date: Fri, 21 Nov 2025 13:14:31 +0530 Subject: [PATCH 4/4] run prettier --- core/llm/llms/Anthropic.ts | 2 +- core/llm/llms/Bedrock.ts | 2 +- core/llm/llms/Gemini.ts | 2 +- core/llm/llms/Ollama.ts | 2 +- core/util/url.ts | 2 +- packages/openai-adapters/src/apis/Anthropic.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index 4d128e91d16..a1fdfbe3598 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -28,9 +28,9 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { DEFAULT_REASONING_TOKENS } from "../constants.js"; import { BaseLLM } from "../index.js"; -import { extractBase64FromDataUrl } from "../../util/url.js"; class Anthropic extends BaseLLM { static providerName = "anthropic"; diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index a3ce15aa00c..7b338e9df59 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -21,11 +21,11 @@ import type { CompletionOptions } from "../../index.js"; import { ChatMessage, Chunk, LLMOptions, MessageContent } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { parseDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js"; import { getSecureID } from "../utils/getSecureID.js"; import { withLLMRetry } from "../utils/retry.js"; -import { parseDataUrl } from "../../util/url.js"; interface ModelConfig { formatPayload: (text: string) => any; diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 2c7ee68b539..0e0b9b243fe 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -11,6 +11,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { GeminiChatContent, @@ -21,7 +22,6 @@ import { GeminiToolFunctionDeclaration, convertContinueToolToGeminiFunction, } from "./gemini-types"; -import { extractBase64FromDataUrl } from "../../util/url.js"; class Gemini extends BaseLLM { static providerName = "gemini"; diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 33712d98d80..7e36ac83647 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -13,8 +13,8 @@ import { } from "../../index.js"; import { renderChatMessage } from "../../util/messageContent.js"; import { getRemoteModelInfo } from "../../util/ollamaHelper.js"; -import { BaseLLM } from "../index.js"; import { extractBase64FromDataUrl } from "../../util/url.js"; +import { BaseLLM } from "../index.js"; type OllamaChatMessage = { role: ChatMessageRole; diff --git a/core/util/url.ts b/core/util/url.ts index 81131c0191f..3bdc0fc6907 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -1,6 +1,6 @@ import { - parseDataUrl as parseDataUrlFromAdapter, extractBase64FromDataUrl as extractBase64FromDataUrlFromAdapter, + parseDataUrl as parseDataUrlFromAdapter, } from "@continuedev/openai-adapters"; export function canParseUrl(url: string): boolean { diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index b922df3f20a..66656957a13 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -25,7 +25,6 @@ import { CompletionUsage, } from "openai/resources/index"; import { ChatCompletionCreateParams } from "openai/resources/index.js"; -import { extractBase64FromDataUrl } from "../util/url.js"; import { AnthropicConfig } from "../types.js"; import { chatChunk, @@ -35,6 +34,7 @@ import { } from "../util.js"; import { EMPTY_CHAT_COMPLETION } from "../util/emptyChatCompletion.js"; import { safeParseArgs } from "../util/parseArgs.js"; +import { extractBase64FromDataUrl } from "../util/url.js"; import { CACHING_STRATEGIES, CachingStrategyName,