Skip to content

fix: improve image URL parsing robustness across providers #8383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+142 −58
Open

fix: improve image URL parsing robustness across providers #8383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions core/llm/llms/Anthropic.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
} from "../../index.js";
import { safeParseToolCallArgs } from "../../tools/parseArgs.js";
import { renderChatMessage, stripImages } from "../../util/messageContent.js";
import { extractBase64FromDataUrl } from "../../util/url.js";
import { DEFAULT_REASONING_TOKENS } from "../constants.js";
import { BaseLLM } from "../index.js";

Expand Down Expand Up @@ -105,14 +106,22 @@ class Anthropic extends BaseLLM {
});
}
} else {
parts.push({
type: "image",
source: {
type: "base64",
media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url),
data: part.imageUrl.url.split(",")[1],
},
});
const base64Data = extractBase64FromDataUrl(part.imageUrl.url);
if (base64Data) {
parts.push({
type: "image",
source: {
type: "base64",
media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url),
data: base64Data,
},
});
} else {
console.warn(
"Anthropic: skipping image with invalid data URL format",
part.imageUrl.url,
);
}
}
}
}
Expand Down
10 changes: 6 additions & 4 deletions core/llm/llms/Bedrock.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type { CompletionOptions } from "../../index.js";
import { ChatMessage, Chunk, LLMOptions, MessageContent } from "../../index.js";
import { safeParseToolCallArgs } from "../../tools/parseArgs.js";
import { renderChatMessage, stripImages } from "../../util/messageContent.js";
import { parseDataUrl } from "../../util/url.js";
import { BaseLLM } from "../index.js";
import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js";
import { getSecureID } from "../utils/getSecureID.js";
Expand Down Expand Up @@ -545,8 +546,9 @@ class Bedrock extends BaseLLM {
if (part.type === "text") {
blocks.push({ text: part.text });
} else if (part.type === "imageUrl" && part.imageUrl) {
try {
const [mimeType, base64Data] = part.imageUrl.url.split(",");
const parsed = parseDataUrl(part.imageUrl.url);
if (parsed) {
const { mimeType, base64Data } = parsed;
const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg";
if (
format === ImageFormat.JPEG ||
Expand All @@ -568,8 +570,8 @@ class Bedrock extends BaseLLM {
part,
);
}
} catch (error) {
console.warn("Bedrock: failed to process image part", error, part);
} else {
console.warn("Bedrock: failed to process image part", part);
}
}
}
Expand Down
36 changes: 26 additions & 10 deletions core/llm/llms/Gemini.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
} from "../../index.js";
import { safeParseToolCallArgs } from "../../tools/parseArgs.js";
import { renderChatMessage, stripImages } from "../../util/messageContent.js";
import { extractBase64FromDataUrl } from "../../util/url.js";
import { BaseLLM } from "../index.js";
import {
GeminiChatContent,
Expand Down Expand Up @@ -184,16 +185,31 @@ class Gemini extends BaseLLM {
}

continuePartToGeminiPart(part: MessagePart): GeminiChatContentPart {
return part.type === "text"
? {
text: part.text,
}
: {
inlineData: {
mimeType: "image/jpeg",
data: part.imageUrl?.url.split(",")[1],
},
};
if (part.type === "text") {
return {
text: part.text,
};
}

let data = "";
if (part.imageUrl?.url) {
const extracted = extractBase64FromDataUrl(part.imageUrl.url);
if (extracted) {
data = extracted;
} else {
console.warn(
"Gemini: skipping image with invalid data URL format",
part.imageUrl.url,
);
}
}

return {
inlineData: {
mimeType: "image/jpeg",
data,
},
};
}

public prepareBody(
Expand Down
10 changes: 9 additions & 1 deletion core/llm/llms/Ollama.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
} from "../../index.js";
import { renderChatMessage } from "../../util/messageContent.js";
import { getRemoteModelInfo } from "../../util/ollamaHelper.js";
import { extractBase64FromDataUrl } from "../../util/url.js";
import { BaseLLM } from "../index.js";

type OllamaChatMessage = {
Expand Down Expand Up @@ -303,9 +304,16 @@ class Ollama extends BaseLLM implements ModelInstaller {
const images: string[] = [];
message.content.forEach((part) => {
if (part.type === "imageUrl" && part.imageUrl) {
const image = part.imageUrl?.url.split(",").at(-1);
const image = part.imageUrl?.url
? extractBase64FromDataUrl(part.imageUrl.url)
Copy link
Contributor

@cubic-dev-ai cubic-dev-ai bot Oct 22, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Calling extractBase64FromDataUrl here will throw on malformed data URLs and bubble up, regressing the previous behavior that tolerated bad input; wrap this in error handling or preserve the non-throwing fallback so malformed URLs don’t crash message conversion.

Prompt for AI agents 
Address the following comment on core/llm/llms/Ollama.ts at line 308:

<comment>Calling extractBase64FromDataUrl here will throw on malformed data URLs and bubble up, regressing the previous behavior that tolerated bad input; wrap this in error handling or preserve the non-throwing fallback so malformed URLs don’t crash message conversion.</comment>

<file context>
@@ -303,7 +304,9 @@ class Ollama extends BaseLLM implements ModelInstaller {
         if (part.type === &quot;imageUrl&quot; &amp;&amp; part.imageUrl) {
-          const image = part.imageUrl?.url.split(&quot;,&quot;).at(-1);
+          const image = part.imageUrl?.url
+            ? extractBase64FromDataUrl(part.imageUrl.url)
+            : undefined;
           if (image) {
</file context>
Fix with Cubic

Copy link
Contributor Author

@dcs-soni dcs-soni Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The extractBase64FromDataUrl function now returns undefined instead of throwing on malformed URLs

Copy link
Contributor

@cubic-dev-ai cubic-dev-ai bot Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the feedback! I've saved this as a new learning to improve future reviews.

: undefined;
if (image) {
images.push(image);
} else if (part.imageUrl?.url) {
console.warn(
"Ollama: skipping image with invalid data URL format",
part.imageUrl.url,
);
}
}
});
Expand Down
39 changes: 39 additions & 0 deletions core/util/url.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,42 @@ export function canParseUrl(url: string): boolean {
return false;
}
}

export function parseDataUrl(dataUrl: string):
| {
mimeType: string;
base64Data: string;
}
| undefined {
const urlParts = dataUrl.split(",");

if (urlParts.length < 2) {
return undefined;
}

const [mimeType, ...base64Parts] = urlParts;
const base64Data = base64Parts.join(",");

return { mimeType, base64Data };
}

export function extractBase64FromDataUrl(dataUrl: string): string | undefined {
return parseDataUrl(dataUrl)?.base64Data;
}

export function safeSplit(
input: string,
delimiter: string,
expectedParts: number,
errorContext: string = "input",
): string[] {
const parts = input.split(delimiter);

if (parts.length !== expectedParts) {
throw new Error(
`Invalid ${errorContext} format: expected ${expectedParts} parts separated by "${delimiter}", got ${parts.length}`,
);
}

return parts;
}
25 changes: 17 additions & 8 deletions packages/openai-adapters/src/apis/Anthropic.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
CompletionUsage,
} from "openai/resources/index";
import { ChatCompletionCreateParams } from "openai/resources/index.js";
import { extractBase64FromDataUrl } from "../../../../core/util/url.js";
import { AnthropicConfig } from "../types.js";
import {
chatChunk,
Expand Down Expand Up @@ -194,14 +195,22 @@ export class AnthropicApi implements BaseLlmApi {
if (part.type === "image_url") {
const dataUrl = part.image_url.url;
if (dataUrl?.startsWith("data:")) {
blocks.push({
type: "image",
source: {
type: "base64",
media_type: getAnthropicMediaTypeFromDataUrl(dataUrl),
data: dataUrl.split(",")[1],
},
});
const base64Data = extractBase64FromDataUrl(dataUrl);
if (base64Data) {
blocks.push({
type: "image",
source: {
type: "base64",
media_type: getAnthropicMediaTypeFromDataUrl(dataUrl),
data: base64Data,
},
});
} else {
console.warn(
"Anthropic: skipping image with invalid data URL format",
dataUrl,
);
}
}
} else {
const text = part.type === "text" ? part.text : part.refusal;
Expand Down
55 changes: 28 additions & 27 deletions packages/openai-adapters/src/apis/Bedrock.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {

import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
import { fromStatic } from "@aws-sdk/token-providers";
import { parseDataUrl } from "../../../../core/util/url.js";
import { BedrockConfig } from "../types.js";
import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js";
import { safeParseArgs } from "../util/parseArgs.js";
Expand Down Expand Up @@ -134,35 +135,35 @@ export class BedrockApi implements BaseLlmApi {
throw new Error("Unsupported part type: input_audio");
case "image_url":
default:
try {
const [mimeType, base64Data] = (
part as ChatCompletionContentPartImage
).image_url.url.split(",");
const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg";
if (
format === ImageFormat.JPEG ||
format === ImageFormat.PNG ||
format === ImageFormat.WEBP ||
format === ImageFormat.GIF
) {
return {
image: {
format,
source: {
bytes: Uint8Array.from(Buffer.from(base64Data, "base64")),
},
},
};
} else {
console.warn(
`Bedrock: skipping unsupported image part format: ${format}`,
);
return { text: "[Unsupported image format]" };
}
} catch (error) {
console.warn("Bedrock: failed to process image part", error);
const parsed = parseDataUrl(
(part as ChatCompletionContentPartImage).image_url.url,
);
if (!parsed) {
console.warn("Bedrock: failed to process image part - invalid URL");
return { text: "[Failed to process image]" };
}
const { mimeType, base64Data } = parsed;
const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg";
if (
format === ImageFormat.JPEG ||
format === ImageFormat.PNG ||
format === ImageFormat.WEBP ||
format === ImageFormat.GIF
) {
return {
image: {
format,
source: {
bytes: Uint8Array.from(Buffer.from(base64Data, "base64")),
},
},
};
} else {
console.warn(
`Bedrock: skipping unsupported image part format: ${format}`,
);
return { text: "[Unsupported image format]" };
}
}
}

Expand Down
Loading