From ffe9a92b156ed63926d7e0095f9c36357082b92a Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 4 Aug 2025 21:06:36 -0700 Subject: [PATCH 01/24] Prototype of using ImageGenerationTool --- .../Image/ImageGenerationTool.cs | 32 ++++++++++ .../OpenAIJsonContext.cs | 1 + .../OpenAIResponsesChatClient.cs | 60 ++++++++++++++++++- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs new file mode 100644 index 00000000000..88f195622c3 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// Represents a hosted tool that can be specified to an AI service to enable it to perform image generation. +/// +/// This tool does not itself implement image generation. It is a marker that can be used to inform a service +/// that the service is allowed to perform image generation if the service is capable of doing so. +/// +[Experimental("MEAI001")] +public class ImageGenerationTool : AITool +{ + /// + /// Initializes a new instance of the class with the specified options. + /// + /// The options to configure the image generation request. If is , default options will be used. + public ImageGenerationTool(TextToImageOptions? options = null) + : base() + { + AdditionalProperties = new AdditionalPropertiesDictionary(new Dictionary + { + [nameof(TextToImageOptions)] = options + }); + } + + /// + public override IReadOnlyDictionary AdditionalProperties { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs index b4f75fe2c94..26d0fa5e856 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs @@ -14,6 +14,7 @@ namespace Microsoft.Extensions.AI; WriteIndented = true)] [JsonSerializable(typeof(OpenAIClientExtensions.ToolJson))] [JsonSerializable(typeof(IDictionary))] +[JsonSerializable(typeof(IDictionary))] [JsonSerializable(typeof(string[]))] [JsonSerializable(typeof(IEnumerable))] [JsonSerializable(typeof(JsonElement))] diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index afb03b518d9..91ff502323b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -12,6 +12,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; +using OpenAI.Images; using OpenAI.Responses; #pragma warning disable S907 // "goto" statement should not be used @@ -349,6 +350,59 @@ internal static ResponseTool ToResponseTool(AIFunction aiFunction, ChatOptions? strict ?? false); } + internal static ResponseTool ToImageResponseTool(ImageGenerationTool imageGenerationTool, ChatOptions? options = null) + { + TextToImageOptions? textToImageOptions = null; + if (imageGenerationTool.AdditionalProperties.TryGetValue(nameof(TextToImageOptions), out object? optionsObj)) + { + textToImageOptions = optionsObj as TextToImageOptions; + } + else if (options?.AdditionalProperties?.TryGetValue(nameof(TextToImageOptions), out object? optionsObj2) ?? false) + { + textToImageOptions = optionsObj2 as TextToImageOptions; + } + + var toolOptions = textToImageOptions?.RawRepresentationFactory?.Invoke(null!) as Dictionary ?? new(); + toolOptions["type"] = "image_generation"; + + // Size: Image dimensions (e.g., 1024x1024, 1024x1536) + if (textToImageOptions?.ImageSize is not null && !toolOptions.ContainsKey("size")) + { + // Use a custom type to ensure the size is formatted correctly. + // This is a workaround for OpenAI's specific size format requirements. + toolOptions["size"] = new GeneratedImageSize( + textToImageOptions.ImageSize.Value.Width, + textToImageOptions.ImageSize.Value.Height).ToString(); + } + + // Quality: Rendering quality (e.g. low, medium, high) --> not exposed + + // Format: File output format + if (textToImageOptions?.MediaType is not null && !toolOptions.ContainsKey("format")) + { + toolOptions["format"] = textToImageOptions.MediaType switch + { + "image/png" => GeneratedImageFileFormat.Png.ToString(), + "image/jpeg" => GeneratedImageFileFormat.Jpeg.ToString(), + "image/webp" => GeneratedImageFileFormat.Webp.ToString(), + _ => string.Empty, + }; + } + + // Compression: Compression level (0-100%) for JPEG and WebP formats --> not exposed + + // Background: Transparent or opaque + if (textToImageOptions?.Background is not null && !toolOptions.ContainsKey("background")) + { + toolOptions["background"] = textToImageOptions.Background; + } + + // Can't create the tool, but we can deserialize it from Json + BinaryData? toolOptionsData = BinaryData.FromBytes( + JsonSerializer.SerializeToUtf8Bytes(toolOptions, OpenAIJsonContext.Default.IDictionaryStringString)); + return ModelReaderWriter.Read(toolOptionsData, ModelReaderWriterOptions.Json)!; + } + /// Creates a from a . private static ChatRole ToChatRole(MessageRole? role) => role switch @@ -403,7 +457,11 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt result.Tools.Add(ToResponseTool(aiFunction, options)); break; - case HostedWebSearchTool webSearchTool: + case ImageGenerationTool imageGenerationTool: + result.Tools.Add(ToImageResponseTool(imageGenerationTool, options)); + break; + + case HostedWebSearchTool: WebSearchUserLocation? location = null; if (webSearchTool.AdditionalProperties.TryGetValue(nameof(WebSearchUserLocation), out object? objLocation)) { From e5edc77bc88dc1018d41f90a07bb6de5d372054a Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Fri, 8 Aug 2025 16:50:39 -0700 Subject: [PATCH 02/24] Handle DataContent returned from ImageGen --- .../OpenAIResponsesChatClient.cs | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 91ff502323b..076c69c7805 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -4,16 +4,20 @@ using System; using System.ClientModel.Primitives; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Net.Mime; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; using OpenAI.Images; using OpenAI.Responses; +using static System.Runtime.InteropServices.JavaScript.JSType; #pragma warning disable S907 // "goto" statement should not be used #pragma warning disable S1067 // Expressions should not be too complex @@ -164,7 +168,15 @@ internal static IEnumerable ToChatMessages(IEnumerable ToChatMessages(IEnumerable? additionalRawData = imageGenResultType + .GetProperty("SerializedAdditionalRawData", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + ?.GetValue(outputItem) as IDictionary; + + string mediaType = "image/png"; + if (additionalRawData?.TryGetValue("output_format", out var outputFormat) == true) + { + var stringJsonTypeInfo = (JsonTypeInfo)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string)); + var outputFormatString = JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo); + mediaType = $"image/{outputFormatString}"; + } + + var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty); + + return new DataContent(resultBytes, mediaType) + { + RawRepresentation = outputItem, + }; + + } + /// public IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) From 2d19cce9631dd229fd4a0757974b9071e533973b Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Fri, 8 Aug 2025 20:47:42 -0700 Subject: [PATCH 03/24] React to rename and improve metadata --- .../Image/ImageGenerationTool.cs | 4 +- .../OpenAIJsonContext.cs | 1 + .../OpenAIResponsesChatClient.cs | 89 +++++++++++-------- 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs index 88f195622c3..4a2dabe5c9b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs @@ -18,12 +18,12 @@ public class ImageGenerationTool : AITool /// Initializes a new instance of the class with the specified options. /// /// The options to configure the image generation request. If is , default options will be used. - public ImageGenerationTool(TextToImageOptions? options = null) + public ImageGenerationTool(ImageGenerationOptions? options = null) : base() { AdditionalProperties = new AdditionalPropertiesDictionary(new Dictionary { - [nameof(TextToImageOptions)] = options + [nameof(ImageGenerationOptions)] = options }); } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs index 26d0fa5e856..4bf13c7971c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs @@ -18,4 +18,5 @@ namespace Microsoft.Extensions.AI; [JsonSerializable(typeof(string[]))] [JsonSerializable(typeof(IEnumerable))] [JsonSerializable(typeof(JsonElement))] +[JsonSerializable(typeof(int))] internal sealed partial class OpenAIJsonContext : JsonSerializerContext; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 076c69c7805..70b7f6266c9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Net.Mime; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; @@ -17,7 +16,6 @@ using Microsoft.Shared.Diagnostics; using OpenAI.Images; using OpenAI.Responses; -using static System.Runtime.InteropServices.JavaScript.JSType; #pragma warning disable S907 // "goto" statement should not be used #pragma warning disable S1067 // Expressions should not be too complex @@ -190,34 +188,55 @@ internal static IEnumerable ToChatMessages(IEnumerable? additionalRawData = imageGenResultType - .GetProperty("SerializedAdditionalRawData", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + .GetProperty("SerializedAdditionalRawData", InternalBindingFlags) ?.GetValue(outputItem) as IDictionary; - string mediaType = "image/png"; - if (additionalRawData?.TryGetValue("output_format", out var outputFormat) == true) - { - var stringJsonTypeInfo = (JsonTypeInfo)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string)); - var outputFormatString = JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo); - mediaType = $"image/{outputFormatString}"; - } + // Properties + // background + // output_format + // quality + // revised_prompt + // size + + string outputFormat = getStringProperty("output_format") ?? "png"; var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty); - return new DataContent(resultBytes, mediaType) + return new DataContent(resultBytes, $"image/{outputFormat}") { RawRepresentation = outputItem, + AdditionalProperties = new() + { + ["background"] = getStringProperty("background"), + ["output_format"] = outputFormat, + ["quality"] = getStringProperty("quality"), + ["revised_prompt"] = getStringProperty("revised_prompt"), + ["size"] = getStringProperty("size"), + ["status"] = imageGenStatus, + } }; + string? getStringProperty(string name) + { + if (additionalRawData?.TryGetValue(name, out var outputFormat) == true) + { + var stringJsonTypeInfo = (JsonTypeInfo)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string)); + return JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo); + } + + return null; + } } /// @@ -397,35 +416,33 @@ internal static ResponseTool ToResponseTool(AIFunction aiFunction, ChatOptions? internal static ResponseTool ToImageResponseTool(ImageGenerationTool imageGenerationTool, ChatOptions? options = null) { - TextToImageOptions? textToImageOptions = null; - if (imageGenerationTool.AdditionalProperties.TryGetValue(nameof(TextToImageOptions), out object? optionsObj)) + ImageGenerationOptions? imageGenerationOptions = null; + if (imageGenerationTool.AdditionalProperties.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj)) { - textToImageOptions = optionsObj as TextToImageOptions; + imageGenerationOptions = optionsObj as ImageGenerationOptions; } - else if (options?.AdditionalProperties?.TryGetValue(nameof(TextToImageOptions), out object? optionsObj2) ?? false) + else if (options?.AdditionalProperties?.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj2) ?? false) { - textToImageOptions = optionsObj2 as TextToImageOptions; + imageGenerationOptions = optionsObj2 as ImageGenerationOptions; } - var toolOptions = textToImageOptions?.RawRepresentationFactory?.Invoke(null!) as Dictionary ?? new(); + var toolOptions = imageGenerationOptions?.RawRepresentationFactory?.Invoke(null!) as Dictionary ?? new(); toolOptions["type"] = "image_generation"; // Size: Image dimensions (e.g., 1024x1024, 1024x1536) - if (textToImageOptions?.ImageSize is not null && !toolOptions.ContainsKey("size")) + if (imageGenerationOptions?.ImageSize is not null && !toolOptions.ContainsKey("size")) { // Use a custom type to ensure the size is formatted correctly. // This is a workaround for OpenAI's specific size format requirements. toolOptions["size"] = new GeneratedImageSize( - textToImageOptions.ImageSize.Value.Width, - textToImageOptions.ImageSize.Value.Height).ToString(); + imageGenerationOptions.ImageSize.Value.Width, + imageGenerationOptions.ImageSize.Value.Height).ToString(); } - // Quality: Rendering quality (e.g. low, medium, high) --> not exposed - // Format: File output format - if (textToImageOptions?.MediaType is not null && !toolOptions.ContainsKey("format")) + if (imageGenerationOptions?.MediaType is not null && !toolOptions.ContainsKey("format")) { - toolOptions["format"] = textToImageOptions.MediaType switch + toolOptions["output_format"] = imageGenerationOptions.MediaType switch { "image/png" => GeneratedImageFileFormat.Png.ToString(), "image/jpeg" => GeneratedImageFileFormat.Jpeg.ToString(), @@ -434,17 +451,19 @@ internal static ResponseTool ToImageResponseTool(ImageGenerationTool imageGenera }; } - // Compression: Compression level (0-100%) for JPEG and WebP formats --> not exposed - - // Background: Transparent or opaque - if (textToImageOptions?.Background is not null && !toolOptions.ContainsKey("background")) - { - toolOptions["background"] = textToImageOptions.Background; - } + // unexposed properties, string unless noted + // background: transparent, opaque, auto + // input_fidelity: effort model exerts to match input (high, low) + // input_image_mask: optional image mask for inpainting. Object with property file_id string or image_url data string. + // model: Model ID to use for image generation + // moderation: Moderation level (auto, low) + // output_compression: (int) Compression level (0-100%) for JPEG and WebP formats + // partial_images: (int) Number of partial images to return (0-3) + // quality: Rendering quality (e.g. low, medium, high) // Can't create the tool, but we can deserialize it from Json BinaryData? toolOptionsData = BinaryData.FromBytes( - JsonSerializer.SerializeToUtf8Bytes(toolOptions, OpenAIJsonContext.Default.IDictionaryStringString)); + JsonSerializer.SerializeToUtf8Bytes(toolOptions, OpenAIJsonContext.Default.IDictionaryStringObject)); return ModelReaderWriter.Read(toolOptionsData, ModelReaderWriterOptions.Json)!; } @@ -506,7 +525,7 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt result.Tools.Add(ToImageResponseTool(imageGenerationTool, options)); break; - case HostedWebSearchTool: + case HostedWebSearchTool webSearchTool: WebSearchUserLocation? location = null; if (webSearchTool.AdditionalProperties.TryGetValue(nameof(WebSearchUserLocation), out object? objLocation)) { From 5eef474ffcea97294c1b6282f97bfc56e3355f3d Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 20 Aug 2025 13:50:12 -0700 Subject: [PATCH 04/24] Handle image_generation tool content from streaming --- .../OpenAIResponsesChatClient.cs | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 70b7f6266c9..4dde2b46684 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -239,6 +239,64 @@ private static DataContent GetContentFromImageGen(ResponseItem outputItem) } } + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(ResponseItem))] + private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResponseUpdate update) + { + const BindingFlags InternalBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; + var partialImageEventType = Type.GetType("OpenAI.Responses.InternalResponseImageGenCallPartialImageEvent, OpenAI"); + if (partialImageEventType == null) + { + throw new InvalidOperationException("Unable to determine the type of the image generation result."); + } + + var imageGenResult = partialImageEventType.GetProperty("PartialImageB64", InternalBindingFlags)?.GetValue(update) as string; + var imageGenItemId = partialImageEventType.GetProperty("ItemId", InternalBindingFlags)?.GetValue(update) as string; + var imageGenOutputIndex = partialImageEventType.GetProperty("OutputIndex", InternalBindingFlags)?.GetValue(update) as int?; + var imageGenPartialImageIndex = partialImageEventType.GetProperty("PartialImageIndex", InternalBindingFlags)?.GetValue(update) as int?; + + IDictionary? additionalRawData = partialImageEventType + .GetProperty("SerializedAdditionalRawData", InternalBindingFlags) + ?.GetValue(update) as IDictionary; + + // Properties + // background + // output_format + // quality + // revised_prompt + // size + + string outputFormat = getStringProperty("output_format") ?? "png"; + + var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty); + + return new DataContent(resultBytes, $"image/{outputFormat}") + { + RawRepresentation = update, + AdditionalProperties = new() + { + ["ItemId"] = imageGenItemId, + ["OutputIndex"] = imageGenOutputIndex, + ["PartialImageIndex"] = imageGenPartialImageIndex, + ["background"] = getStringProperty("background"), + ["output_format"] = outputFormat, + ["quality"] = getStringProperty("quality"), + ["revised_prompt"] = getStringProperty("revised_prompt"), + ["size"] = getStringProperty("size"), + } + }; + + string? getStringProperty(string name) + { + if (additionalRawData?.TryGetValue(name, out var outputFormat) == true) + { + var stringJsonTypeInfo = (JsonTypeInfo)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string)); + return JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo); + } + + return null; + } + } + /// public IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) @@ -389,7 +447,16 @@ ChatResponseUpdate CreateUpdate(AIContent? content = null) => break; default: - yield return CreateUpdate(); + + if (streamingUpdate.GetType().Name == "InternalResponseImageGenCallPartialImageEvent") + { + yield return CreateUpdate(GetContentFromImageGenPartialImageEvent(streamingUpdate)); + } + else + { + yield return CreateUpdate(); + } + break; } } From ff808043cc181dac6c0a6ae998bb30e450b0709e Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 25 Aug 2025 12:16:24 -0700 Subject: [PATCH 05/24] Add handling for combining updates with images --- .../ChatCompletion/ChatResponseExtensions.cs | 251 ++++++++++++++++++ .../ChatCompletion/ChatResponseUpdate.cs | 2 +- .../ChatResponseUpdateCoalescingOptions.cs | 30 +++ 3 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs index e41368f115d..15c89daae8b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs @@ -13,6 +13,7 @@ #pragma warning disable S109 // Magic numbers should not be used #pragma warning disable S1121 // Assignments should not be made from within sub-expressions +#pragma warning disable EXTAI0001 // Suppress experimental warnings for internal usage namespace Microsoft.Extensions.AI; @@ -68,6 +69,32 @@ public static void AddMessages(this IList list, IEnumerableConverts the into instances and adds them to . + /// The destination list to which the newly constructed messages should be added. + /// The instances to convert to messages and add to the list. + /// Options for configuring how the updates are coalesced. + /// is . + /// is . + /// + /// As part of combining into a series of instances, the + /// method may use to determine message boundaries, as well as coalesce + /// contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static void AddMessages(this IList list, IEnumerable updates, ChatResponseUpdateCoalescingOptions? options) + { + _ = Throw.IfNull(list); + _ = Throw.IfNull(updates); + + if (updates is ICollection { Count: 0 }) + { + return; + } + + list.AddMessages(updates.ToChatResponse(options)); + } + /// Converts the into a instance and adds it to . /// The destination list to which the newly constructed message should be added. /// The instance to convert to a message and add to the list. @@ -122,6 +149,128 @@ static async Task AddMessagesAsync( list.AddMessages(await updates.ToChatResponseAsync(cancellationToken).ConfigureAwait(false)); } + /// Converts the into instances and adds them to . + /// The list to which the newly constructed messages should be added. + /// The instances to convert to messages and add to the list. + /// Options for configuring how the updates are coalesced. + /// The to monitor for cancellation requests. The default is . + /// A representing the completion of the operation. + /// is . + /// is . + /// + /// As part of combining into a series of instances, tne + /// method may use to determine message boundaries, as well as coalesce + /// contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static Task AddMessagesAsync( + this IList list, IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(list); + _ = Throw.IfNull(updates); + + return AddMessagesAsync(list, updates, options, cancellationToken); + + static async Task AddMessagesAsync( + IList list, IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken) => + list.AddMessages(await updates.ToChatResponseAsync(options, cancellationToken).ConfigureAwait(false)); + } + + /// Applies a to an existing . + /// The response to which the update should be applied. + /// The update to apply to the response. + /// Options for configuring how the update is applied. + /// is . + /// is . + /// + /// This method modifies the existing by incorporating the content and metadata + /// from the . This includes using to determine + /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static void ApplyUpdate(this ChatResponse response, ChatResponseUpdate update, ChatResponseUpdateCoalescingOptions? options = null) + { + _ = Throw.IfNull(response); + _ = Throw.IfNull(update); + + ProcessUpdate(update, response, options); + FinalizeResponse(response); + } + + /// Applies instances to an existing . + /// The response to which the updates should be applied. + /// The updates to apply to the response. + /// Options for configuring how the updates are applied. + /// is . + /// is . + /// + /// This method modifies the existing by incorporating the content and metadata + /// from the . This includes using to determine + /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static void ApplyUpdates(this ChatResponse response, IEnumerable updates, ChatResponseUpdateCoalescingOptions? options = null) + { + _ = Throw.IfNull(response); + _ = Throw.IfNull(updates); + + if (updates is ICollection { Count: 0 }) + { + return; + } + + foreach (var update in updates) + { + ProcessUpdate(update, response, options); + } + + FinalizeResponse(response); + } + + /// Applies instances to an existing asynchronously. + /// The response to which the updates should be applied. + /// The updates to apply to the response. + /// Options for configuring how the updates are applied. + /// The to monitor for cancellation requests. The default is . + /// A representing the completion of the operation. + /// is . + /// is . + /// + /// This method modifies the existing by incorporating the content and metadata + /// from the . This includes using to determine + /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static Task ApplyUpdatesAsync( + this ChatResponse response, + IAsyncEnumerable updates, + ChatResponseUpdateCoalescingOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(response); + _ = Throw.IfNull(updates); + + return ApplyUpdatesAsync(response, updates, options, cancellationToken); + + static async Task ApplyUpdatesAsync( + ChatResponse response, + IAsyncEnumerable updates, + ChatResponseUpdateCoalescingOptions? options, + CancellationToken cancellationToken) + { + await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + ProcessUpdate(update, response, options); + } + + FinalizeResponse(response); + } + } + /// Combines instances into a single . /// The updates to be combined. /// The combined . @@ -149,6 +298,35 @@ public static ChatResponse ToChatResponse( return response; } + /// Combines instances into a single . + /// The updates to be combined. + /// Options for configuring how the updates are coalesced. + /// The combined . + /// is . + /// + /// As part of combining into a single , the method will attempt to reconstruct + /// instances. This includes using to determine + /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static ChatResponse ToChatResponse( + this IEnumerable updates, ChatResponseUpdateCoalescingOptions? options) + { + _ = Throw.IfNull(updates); + + ChatResponse response = new(); + + foreach (var update in updates) + { + ProcessUpdate(update, response, options); + } + + FinalizeResponse(response); + + return response; + } + /// Combines instances into a single . /// The updates to be combined. /// The to monitor for cancellation requests. The default is . @@ -183,6 +361,42 @@ static async Task ToChatResponseAsync( } } + /// Combines instances into a single . + /// The updates to be combined. + /// Options for configuring how the updates are coalesced. + /// The to monitor for cancellation requests. The default is . + /// The combined . + /// is . + /// + /// As part of combining into a single , the method will attempt to reconstruct + /// instances. This includes using to determine + /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple + /// instances in a row may be combined into a single . + /// + [Experimental("EXTAI0001")] + public static Task ToChatResponseAsync( + this IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(updates); + + return ToChatResponseAsync(updates, options, cancellationToken); + + static async Task ToChatResponseAsync( + IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken) + { + ChatResponse response = new(); + + await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + ProcessUpdate(update, response, options); + } + + FinalizeResponse(response); + + return response; + } + } + /// Coalesces sequential content elements. internal static void CoalesceTextContent(IList contents) { @@ -301,6 +515,15 @@ private static void FinalizeResponse(ChatResponse response) /// The update to process. /// The object that should be updated based on . private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse response) + { + ProcessUpdate(update, response, null); + } + + /// Processes the , incorporating its contents into . + /// The update to process. + /// The object that should be updated based on . + /// Options for configuring how the updates are coalesced. + private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse response, ChatResponseUpdateCoalescingOptions? options) { // If there is no message created yet, or if the last update we saw had a different // message ID than the newest update, create a new message. @@ -363,6 +586,34 @@ private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse respon (response.Usage ??= new()).Add(usage.Details); break; + case DataContent dataContent when options is not null && + options.ReplaceDataContentWithSameName && + !string.IsNullOrEmpty(dataContent.Name): + // Check if there's an existing DataContent with the same name to replace + int existingIndex = -1; + for (int i = 0; i < message.Contents.Count; i++) + { + if (message.Contents[i] is DataContent existingDataContent && + string.Equals(existingDataContent.Name, dataContent.Name, StringComparison.Ordinal)) + { + existingIndex = i; + break; + } + } + + if (existingIndex >= 0) + { + // Replace the existing DataContent + message.Contents[existingIndex] = dataContent; + } + else + { + // No existing DataContent with the same name, add it normally + message.Contents.Add(dataContent); + } + + break; + default: message.Contents.Add(content); break; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs index bea91f97ed9..7b27bbf126c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs @@ -22,7 +22,7 @@ namespace Microsoft.Extensions.AI; /// /// /// The relationship between and is -/// codified in the and +/// codified in the and /// , which enable bidirectional conversions /// between the two. Note, however, that the provided conversions may be lossy, for example if multiple /// updates all have different objects whereas there's only one slot for diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs new file mode 100644 index 00000000000..9671d4ea43b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides options for configuring how instances are coalesced +/// when converting them to instances. +/// +[Experimental("EXTAI0001")] +public class ChatResponseUpdateCoalescingOptions +{ + /// + /// Gets or sets a value indicating whether to replace existing items + /// when a new item with the same is encountered. + /// + /// + /// to replace existing items with the same name; + /// to keep all items. The default is . + /// + /// + /// When this property is , if a item is being added + /// and there's already a item in the content list with the same + /// , the existing item will be replaced with the new one. + /// This is useful for scenarios where updated data should override previous data with the same identifier. + /// + public bool ReplaceDataContentWithSameName { get; set; } +} From 1725ce1d67c5dfc4091a225072a8d8af43aaadd4 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 25 Aug 2025 18:03:59 -0700 Subject: [PATCH 06/24] Add tests for new ChatResponseUpdateExtensions --- .../ChatResponseUpdateExtensionsTests.cs | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs index 45a82542da8..ae31c0fccad 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs @@ -10,6 +10,7 @@ using Xunit; #pragma warning disable SA1204 // Static elements should appear before instance elements +#pragma warning disable EXTAI0001 // Suppress experimental warnings for testing namespace Microsoft.Extensions.AI; @@ -21,6 +22,340 @@ public void InvalidArgs_Throws() Assert.Throws("updates", () => ((List)null!).ToChatResponse()); } + [Fact] + public void ApplyUpdate_InvalidArgs_Throws() + { + var response = new ChatResponse(); + var update = new ChatResponseUpdate(); + + Assert.Throws("response", () => ((ChatResponse)null!).ApplyUpdate(update)); + Assert.Throws("update", () => response.ApplyUpdate(null!)); + } + + [Fact] + public void ApplyUpdate_WithDefaultOptions_UpdatesResponse() + { + // Arrange + var response = new ChatResponse(); + var update = new ChatResponseUpdate(ChatRole.Assistant, "Hello, world!") + { + ResponseId = "resp123", + MessageId = "msg456", + CreatedAt = new DateTimeOffset(2024, 1, 1, 12, 0, 0, TimeSpan.Zero), + ModelId = "model789", + ConversationId = "conv101112", + FinishReason = ChatFinishReason.Stop + }; + + // Act + response.ApplyUpdate(update); + + // Assert + Assert.Equal("resp123", response.ResponseId); + Assert.Equal("model789", response.ModelId); + Assert.Equal(new DateTimeOffset(2024, 1, 1, 12, 0, 0, TimeSpan.Zero), response.CreatedAt); + Assert.Equal("conv101112", response.ConversationId); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + + var message = Assert.Single(response.Messages); + Assert.Equal("msg456", message.MessageId); + Assert.Equal(ChatRole.Assistant, message.Role); + Assert.Equal("Hello, world!", message.Text); + } + + [Fact] + public void ApplyUpdate_WithOptions_UsesCoalescingOptions() + { + // Arrange + var response = new ChatResponse(); + var existingDataContent = new DataContent("data:text/plain;base64,aGVsbG8=") + { + Name = "test-data" + }; + response.Messages.Add(new ChatMessage(ChatRole.Assistant, [existingDataContent])); + + var newDataContent = new DataContent("data:text/plain;base64,d29ybGQ=") + { + Name = "test-data" // Same name as existing + }; + var update = new ChatResponseUpdate + { + Contents = [newDataContent] + }; + + var options = new ChatResponseUpdateCoalescingOptions + { + ReplaceDataContentWithSameName = true + }; + + // Act + response.ApplyUpdate(update, options); + + // Assert + var message = Assert.Single(response.Messages); + var dataContent = Assert.Single(message.Contents.OfType()); + Assert.Equal("data:text/plain;base64,d29ybGQ=", dataContent.Uri); + Assert.Equal("test-data", dataContent.Name); + } + + [Fact] + public void ApplyUpdate_WithNullOptions_WorksCorrectly() + { + // Arrange + var response = new ChatResponse(); + var update = new ChatResponseUpdate(ChatRole.User, "Test message"); + + // Act - explicitly pass null options + response.ApplyUpdate(update, null); + + // Assert + var message = Assert.Single(response.Messages); + Assert.Equal("Test message", message.Text); + Assert.Equal(ChatRole.User, message.Role); + } + + [Fact] + public void ApplyUpdates_InvalidArgs_Throws() + { + var response = new ChatResponse(); + var updates = new List(); + + Assert.Throws("response", () => ((ChatResponse)null!).ApplyUpdates(updates)); + Assert.Throws("updates", () => response.ApplyUpdates(null!)); + } + + [Fact] + public void ApplyUpdates_WithDefaultOptions_UpdatesResponse() + { + // Arrange + var response = new ChatResponse(); + var updates = new List + { + new(ChatRole.Assistant, "Hello") { MessageId = "msg1", ResponseId = "resp1" }, + new(null, ", ") { MessageId = "msg1" }, + new(null, "world!") { MessageId = "msg1", CreatedAt = new DateTimeOffset(2024, 1, 1, 10, 0, 0, TimeSpan.Zero) }, + new() { Contents = [new UsageContent(new() { InputTokenCount = 10, OutputTokenCount = 5 })] } + }; + + // Act + response.ApplyUpdates(updates); + + // Assert + Assert.Equal("resp1", response.ResponseId); + Assert.NotNull(response.Usage); + Assert.Equal(10, response.Usage.InputTokenCount); + Assert.Equal(5, response.Usage.OutputTokenCount); + + var message = Assert.Single(response.Messages); + Assert.Equal("msg1", message.MessageId); + Assert.Equal("Hello, world!", message.Text); + Assert.Equal(new DateTimeOffset(2024, 1, 1, 10, 0, 0, TimeSpan.Zero), message.CreatedAt); + } + + [Fact] + public void ApplyUpdates_WithOptions_UsesCoalescingOptions() + { + // Arrange + var response = new ChatResponse(); + var existingDataContent = new DataContent("data:text/plain;base64,aGVsbG8=") + { + Name = "shared-name" + }; + response.Messages.Add(new ChatMessage(ChatRole.Assistant, [existingDataContent])); + + var sharedNameDataContent = new DataContent("data:text/plain;base64,dXBkYXRl") + { + Name = "shared-name" // Same name + }; + var otherNameDataContent = new DataContent("data:text/plain;base64,b3RoZXI=") + { + Name = "other-name" // Different name + }; + var updates = new List + { + new() { Contents = [sharedNameDataContent] }, + new() { Contents = [otherNameDataContent] } + }; + + var options = new ChatResponseUpdateCoalescingOptions + { + ReplaceDataContentWithSameName = true + }; + + // Act + response.ApplyUpdates(updates, options); + + // Assert + var message = Assert.Single(response.Messages); + Assert.Equal(2, message.Contents.Count); + + var sharedNameContent = message.Contents.OfType().First(c => c.Name == "shared-name"); + Assert.Equal("data:text/plain;base64,dXBkYXRl", sharedNameContent.Uri); // Should be replaced + + var otherNameContent = message.Contents.OfType().First(c => c.Name == "other-name"); + Assert.Equal("data:text/plain;base64,b3RoZXI=", otherNameContent.Uri); // Should be added + } + + [Fact] + public void ApplyUpdates_WithEmptyCollection_DoesNothing() + { + // Arrange + var response = new ChatResponse(); + var updates = new List(); + + // Act + response.ApplyUpdates(updates); + + // Assert + Assert.Empty(response.Messages); + } + + [Fact] + public void ApplyUpdates_WithNullOptions_WorksCorrectly() + { + // Arrange + var response = new ChatResponse(); + var sharedNameDataContent = new DataContent("data:text/plain;base64,dXBkYXRl") + { + Name = "shared-name" // Same name + }; + var updates = new List + { + new(ChatRole.System, [sharedNameDataContent]), + new(ChatRole.System, [sharedNameDataContent]) + }; + + // Act - explicitly pass null options + response.ApplyUpdates(updates, null); + + // Assert + Assert.Single(response.Messages); + Assert.All(response.Messages[0].Contents, c => Assert.IsType(c)); + } + + [Fact] + public async Task ApplyUpdatesAsync_InvalidArgs_Throws() + { + var response = new ChatResponse(); + var updates = YieldAsync(new List()); + + await Assert.ThrowsAsync("response", () => ((ChatResponse)null!).ApplyUpdatesAsync(updates)); + await Assert.ThrowsAsync("updates", () => response.ApplyUpdatesAsync(null!)); + } + + [Fact] + public async Task ApplyUpdatesAsync_WithDefaultOptions_UpdatesResponse() + { + // Arrange + var response = new ChatResponse(); + var updates = new List + { + new(ChatRole.Assistant, "Hello") { MessageId = "msg1", ResponseId = "resp1" }, + new(null, " async") { MessageId = "msg1" }, + new(null, " world!") { MessageId = "msg1", CreatedAt = new DateTimeOffset(2024, 1, 1, 11, 0, 0, TimeSpan.Zero) }, + new() { Contents = [new UsageContent(new() { InputTokenCount = 15, OutputTokenCount = 8 })] } + }; + + // Act + await response.ApplyUpdatesAsync(YieldAsync(updates)); + + // Assert + Assert.Equal("resp1", response.ResponseId); + Assert.NotNull(response.Usage); + Assert.Equal(15, response.Usage.InputTokenCount); + Assert.Equal(8, response.Usage.OutputTokenCount); + + var message = Assert.Single(response.Messages); + Assert.Equal("msg1", message.MessageId); + Assert.Equal("Hello async world!", message.Text); + Assert.Equal(new DateTimeOffset(2024, 1, 1, 11, 0, 0, TimeSpan.Zero), message.CreatedAt); + } + + [Fact] + public async Task ApplyUpdatesAsync_WithOptions_UsesCoalescingOptions() + { + // Arrange + var response = new ChatResponse(); + var existingDataContent = new DataContent("data:text/plain;base64,b3JpZ2luYWw=") + { + Name = "data-key" + }; + response.Messages.Add(new ChatMessage(ChatRole.Assistant, [existingDataContent])); + + var newDataContent = new DataContent("data:text/plain;base64,bmV3VmFsdWU=") + { + Name = "data-key" // Same name + }; + var updates = new List + { + new() { Contents = [newDataContent] }, + }; + + var options = new ChatResponseUpdateCoalescingOptions + { + ReplaceDataContentWithSameName = true + }; + + // Act + await response.ApplyUpdatesAsync(YieldAsync(updates), options); + + // Assert + var message = Assert.Single(response.Messages); + var dataContent = Assert.Single(message.Contents.OfType()); + Assert.Equal("data:text/plain;base64,bmV3VmFsdWU=", dataContent.Uri); // Should be replaced + Assert.Equal("data-key", dataContent.Name); + } + + [Fact] + public async Task ApplyUpdatesAsync_WithNullOptions_WorksCorrectly() + { + // Arrange + var response = new ChatResponse(); + var updates = new List + { + new(ChatRole.Assistant, "Async message with null options") + }; + + // Act - explicitly pass null options + await response.ApplyUpdatesAsync(YieldAsync(updates), null); + + // Assert + var message = Assert.Single(response.Messages); + Assert.Equal("Async message with null options", message.Text); + } + + [Fact] + public async Task ApplyUpdatesAsync_MultipleMessages_ProcessedCorrectly() + { + // Arrange + var response = new ChatResponse(); + var updates = new List + { + new(ChatRole.Assistant, "First") { MessageId = "msg1" }, + new(null, " message") { MessageId = "msg1" }, + new(ChatRole.User, "Second") { MessageId = "msg2" }, + new(null, " message") { MessageId = "msg2" }, + new(ChatRole.Assistant, "Third message") { MessageId = "msg3" } + }; + + // Act + await response.ApplyUpdatesAsync(YieldAsync(updates)); + + // Assert + Assert.Equal(3, response.Messages.Count); + Assert.Equal("First message", response.Messages[0].Text); + Assert.Equal(ChatRole.Assistant, response.Messages[0].Role); + Assert.Equal("msg1", response.Messages[0].MessageId); + + Assert.Equal("Second message", response.Messages[1].Text); + Assert.Equal(ChatRole.User, response.Messages[1].Role); + Assert.Equal("msg2", response.Messages[1].MessageId); + + Assert.Equal("Third message", response.Messages[2].Text); + Assert.Equal(ChatRole.Assistant, response.Messages[2].Role); + Assert.Equal("msg3", response.Messages[2].MessageId); + } + [Theory] [InlineData(false)] [InlineData(true)] From b4fe94ba2a3eb152fb2c3087cae6b2598f33c361 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 19 Sep 2025 17:34:35 -0700 Subject: [PATCH 07/24] Rename ImageGenerationTool to HostedImageGenerationTool --- .../Image/ImageGenerationTool.cs | 32 ------------------- .../Tools/HostedImageGenerationTool.cs | 28 ++++++++++++++++ .../OpenAIResponsesChatClient.cs | 5 +-- 3 files changed, 31 insertions(+), 34 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs deleted file mode 100644 index 4a2dabe5c9b..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationTool.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.AI; - -/// Represents a hosted tool that can be specified to an AI service to enable it to perform image generation. -/// -/// This tool does not itself implement image generation. It is a marker that can be used to inform a service -/// that the service is allowed to perform image generation if the service is capable of doing so. -/// -[Experimental("MEAI001")] -public class ImageGenerationTool : AITool -{ - /// - /// Initializes a new instance of the class with the specified options. - /// - /// The options to configure the image generation request. If is , default options will be used. - public ImageGenerationTool(ImageGenerationOptions? options = null) - : base() - { - AdditionalProperties = new AdditionalPropertiesDictionary(new Dictionary - { - [nameof(ImageGenerationOptions)] = options - }); - } - - /// - public override IReadOnlyDictionary AdditionalProperties { get; } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs new file mode 100644 index 00000000000..897fd4f9f89 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI.Tools; + +/// Represents a hosted tool that can be specified to an AI service to enable it to perform image generation. +/// +/// This tool does not itself implement image generation. It is a marker that can be used to inform a service +/// that the service is allowed to perform image generation if the service is capable of doing so. +/// +[Experimental("MEAI001")] +public class HostedImageGenerationTool : AITool +{ + /// + /// Initializes a new instance of the class with the specified options. + /// + public HostedImageGenerationTool() + : base() + { + } + + /// + /// Gets or sets the options used to configure image generation. + /// + public ImageGenerationOptions? Options { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index b982195db51..452d63a182a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -13,6 +13,7 @@ using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.AI.Tools; using Microsoft.Shared.Diagnostics; using OpenAI.Images; using OpenAI.Responses; @@ -518,7 +519,7 @@ internal static FunctionTool ToResponseTool(AIFunctionDeclaration aiFunction, Ch aiFunction.Description); } - internal static ResponseTool ToImageResponseTool(ImageGenerationTool imageGenerationTool, ChatOptions? options = null) + internal static ResponseTool ToImageResponseTool(HostedImageGenerationTool imageGenerationTool, ChatOptions? options = null) { ImageGenerationOptions? imageGenerationOptions = null; if (imageGenerationTool.AdditionalProperties.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj)) @@ -629,7 +630,7 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt result.Tools.Add(ToResponseTool(aiFunction, options)); break; - case ImageGenerationTool imageGenerationTool: + case HostedImageGenerationTool imageGenerationTool: result.Tools.Add(ToImageResponseTool(imageGenerationTool, options)); break; From 06bfa3008438c34dd80ea55787a91edb500a5c25 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 19 Sep 2025 17:42:25 -0700 Subject: [PATCH 08/24] Remove ChatResponseUpdateCoalescingOptions --- .../ChatCompletion/ChatResponseExtensions.cs | 167 ++---------------- .../ChatResponseUpdateCoalescingOptions.cs | 30 ---- .../ChatResponseUpdateExtensionsTests.cs | 151 +--------------- 3 files changed, 19 insertions(+), 329 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs index 7ea94974e9b..425b2b06b48 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs @@ -13,7 +13,6 @@ #pragma warning disable S109 // Magic numbers should not be used #pragma warning disable S1121 // Assignments should not be made from within sub-expressions -#pragma warning disable EXTAI0001 // Suppress experimental warnings for internal usage namespace Microsoft.Extensions.AI; @@ -69,32 +68,6 @@ public static void AddMessages(this IList list, IEnumerableConverts the into instances and adds them to . - /// The destination list to which the newly constructed messages should be added. - /// The instances to convert to messages and add to the list. - /// Options for configuring how the updates are coalesced. - /// is . - /// is . - /// - /// As part of combining into a series of instances, the - /// method may use to determine message boundaries, as well as coalesce - /// contiguous items where applicable, e.g. multiple - /// instances in a row may be combined into a single . - /// - [Experimental("EXTAI0001")] - public static void AddMessages(this IList list, IEnumerable updates, ChatResponseUpdateCoalescingOptions? options) - { - _ = Throw.IfNull(list); - _ = Throw.IfNull(updates); - - if (updates is ICollection { Count: 0 }) - { - return; - } - - list.AddMessages(updates.ToChatResponse(options)); - } - /// Converts the into a instance and adds it to . /// The destination list to which the newly constructed message should be added. /// The instance to convert to a message and add to the list. @@ -149,38 +122,9 @@ static async Task AddMessagesAsync( list.AddMessages(await updates.ToChatResponseAsync(cancellationToken).ConfigureAwait(false)); } - /// Converts the into instances and adds them to . - /// The list to which the newly constructed messages should be added. - /// The instances to convert to messages and add to the list. - /// Options for configuring how the updates are coalesced. - /// The to monitor for cancellation requests. The default is . - /// A representing the completion of the operation. - /// is . - /// is . - /// - /// As part of combining into a series of instances, tne - /// method may use to determine message boundaries, as well as coalesce - /// contiguous items where applicable, e.g. multiple - /// instances in a row may be combined into a single . - /// - [Experimental("EXTAI0001")] - public static Task AddMessagesAsync( - this IList list, IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(list); - _ = Throw.IfNull(updates); - - return AddMessagesAsync(list, updates, options, cancellationToken); - - static async Task AddMessagesAsync( - IList list, IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken) => - list.AddMessages(await updates.ToChatResponseAsync(options, cancellationToken).ConfigureAwait(false)); - } - /// Applies a to an existing . /// The response to which the update should be applied. /// The update to apply to the response. - /// Options for configuring how the update is applied. /// is . /// is . /// @@ -189,20 +133,19 @@ static async Task AddMessagesAsync( /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple /// instances in a row may be combined into a single . /// - [Experimental("EXTAI0001")] - public static void ApplyUpdate(this ChatResponse response, ChatResponseUpdate update, ChatResponseUpdateCoalescingOptions? options = null) + [Experimental("MEAI0001")] + public static void ApplyUpdate(this ChatResponse response, ChatResponseUpdate update) { _ = Throw.IfNull(response); _ = Throw.IfNull(update); - ProcessUpdate(update, response, options); + ProcessUpdate(update, response); FinalizeResponse(response); } /// Applies instances to an existing . /// The response to which the updates should be applied. /// The updates to apply to the response. - /// Options for configuring how the updates are applied. /// is . /// is . /// @@ -211,8 +154,8 @@ public static void ApplyUpdate(this ChatResponse response, ChatResponseUpdate up /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple /// instances in a row may be combined into a single . /// - [Experimental("EXTAI0001")] - public static void ApplyUpdates(this ChatResponse response, IEnumerable updates, ChatResponseUpdateCoalescingOptions? options = null) + [Experimental("MEAI0001")] + public static void ApplyUpdates(this ChatResponse response, IEnumerable updates) { _ = Throw.IfNull(response); _ = Throw.IfNull(updates); @@ -224,7 +167,7 @@ public static void ApplyUpdates(this ChatResponse response, IEnumerableApplies instances to an existing asynchronously. /// The response to which the updates should be applied. /// The updates to apply to the response. - /// Options for configuring how the updates are applied. /// The to monitor for cancellation requests. The default is . /// A representing the completion of the operation. /// is . @@ -244,27 +186,25 @@ public static void ApplyUpdates(this ChatResponse response, IEnumerable items where applicable, e.g. multiple /// instances in a row may be combined into a single . /// - [Experimental("EXTAI0001")] + [Experimental("MEAI0001")] public static Task ApplyUpdatesAsync( this ChatResponse response, IAsyncEnumerable updates, - ChatResponseUpdateCoalescingOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(response); _ = Throw.IfNull(updates); - return ApplyUpdatesAsync(response, updates, options, cancellationToken); + return ApplyUpdatesAsync(response, updates, cancellationToken); static async Task ApplyUpdatesAsync( ChatResponse response, IAsyncEnumerable updates, - ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken) { await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) { - ProcessUpdate(update, response, options); + ProcessUpdate(update, response); } FinalizeResponse(response); @@ -298,35 +238,6 @@ public static ChatResponse ToChatResponse( return response; } - /// Combines instances into a single . - /// The updates to be combined. - /// Options for configuring how the updates are coalesced. - /// The combined . - /// is . - /// - /// As part of combining into a single , the method will attempt to reconstruct - /// instances. This includes using to determine - /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple - /// instances in a row may be combined into a single . - /// - [Experimental("EXTAI0001")] - public static ChatResponse ToChatResponse( - this IEnumerable updates, ChatResponseUpdateCoalescingOptions? options) - { - _ = Throw.IfNull(updates); - - ChatResponse response = new(); - - foreach (var update in updates) - { - ProcessUpdate(update, response, options); - } - - FinalizeResponse(response); - - return response; - } - /// Combines instances into a single . /// The updates to be combined. /// The to monitor for cancellation requests. The default is . @@ -361,42 +272,6 @@ static async Task ToChatResponseAsync( } } - /// Combines instances into a single . - /// The updates to be combined. - /// Options for configuring how the updates are coalesced. - /// The to monitor for cancellation requests. The default is . - /// The combined . - /// is . - /// - /// As part of combining into a single , the method will attempt to reconstruct - /// instances. This includes using to determine - /// message boundaries, as well as coalescing contiguous items where applicable, e.g. multiple - /// instances in a row may be combined into a single . - /// - [Experimental("EXTAI0001")] - public static Task ToChatResponseAsync( - this IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(updates); - - return ToChatResponseAsync(updates, options, cancellationToken); - - static async Task ToChatResponseAsync( - IAsyncEnumerable updates, ChatResponseUpdateCoalescingOptions? options, CancellationToken cancellationToken) - { - ChatResponse response = new(); - - await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - ProcessUpdate(update, response, options); - } - - FinalizeResponse(response); - - return response; - } - } - /// Coalesces sequential content elements. internal static void CoalesceTextContent(IList contents) { @@ -521,15 +396,6 @@ private static void FinalizeResponse(ChatResponse response) /// The update to process. /// The object that should be updated based on . private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse response) - { - ProcessUpdate(update, response, null); - } - - /// Processes the , incorporating its contents into . - /// The update to process. - /// The object that should be updated based on . - /// Options for configuring how the updates are coalesced. - private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse response, ChatResponseUpdateCoalescingOptions? options) { // If there is no message created yet, or if the last update we saw had a different // message ID or role than the newest update, create a new message. @@ -598,27 +464,22 @@ private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse respon (response.Usage ??= new()).Add(usage.Details); break; - case DataContent dataContent when options is not null && - options.ReplaceDataContentWithSameName && + case DataContent dataContent when !string.IsNullOrEmpty(dataContent.Name): // Check if there's an existing DataContent with the same name to replace - int existingIndex = -1; for (int i = 0; i < message.Contents.Count; i++) { if (message.Contents[i] is DataContent existingDataContent && string.Equals(existingDataContent.Name, dataContent.Name, StringComparison.Ordinal)) { - existingIndex = i; + // Replace the existing DataContent + message.Contents[i] = dataContent; + dataContent = null!; break; } } - if (existingIndex >= 0) - { - // Replace the existing DataContent - message.Contents[existingIndex] = dataContent; - } - else + if (dataContent is not null) { // No existing DataContent with the same name, add it normally message.Contents.Add(dataContent); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs deleted file mode 100644 index 9671d4ea43b..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdateCoalescingOptions.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.AI; - -/// -/// Provides options for configuring how instances are coalesced -/// when converting them to instances. -/// -[Experimental("EXTAI0001")] -public class ChatResponseUpdateCoalescingOptions -{ - /// - /// Gets or sets a value indicating whether to replace existing items - /// when a new item with the same is encountered. - /// - /// - /// to replace existing items with the same name; - /// to keep all items. The default is . - /// - /// - /// When this property is , if a item is being added - /// and there's already a item in the content list with the same - /// , the existing item will be replaced with the new one. - /// This is useful for scenarios where updated data should override previous data with the same identifier. - /// - public bool ReplaceDataContentWithSameName { get; set; } -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs index 456fd788182..71ec4dde2d4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs @@ -10,7 +10,7 @@ using Xunit; #pragma warning disable SA1204 // Static elements should appear before instance elements -#pragma warning disable EXTAI0001 // Suppress experimental warnings for testing +#pragma warning disable MEAI0001 // Suppress experimental warnings for testing namespace Microsoft.Extensions.AI; @@ -33,7 +33,7 @@ public void ApplyUpdate_InvalidArgs_Throws() } [Fact] - public void ApplyUpdate_WithDefaultOptions_UpdatesResponse() + public void ApplyUpdate_UpdatesResponse() { // Arrange var response = new ChatResponse(); @@ -83,13 +83,8 @@ public void ApplyUpdate_WithOptions_UsesCoalescingOptions() Contents = [newDataContent] }; - var options = new ChatResponseUpdateCoalescingOptions - { - ReplaceDataContentWithSameName = true - }; - // Act - response.ApplyUpdate(update, options); + response.ApplyUpdate(update); // Assert var message = Assert.Single(response.Messages); @@ -98,22 +93,6 @@ public void ApplyUpdate_WithOptions_UsesCoalescingOptions() Assert.Equal("test-data", dataContent.Name); } - [Fact] - public void ApplyUpdate_WithNullOptions_WorksCorrectly() - { - // Arrange - var response = new ChatResponse(); - var update = new ChatResponseUpdate(ChatRole.User, "Test message"); - - // Act - explicitly pass null options - response.ApplyUpdate(update, null); - - // Assert - var message = Assert.Single(response.Messages); - Assert.Equal("Test message", message.Text); - Assert.Equal(ChatRole.User, message.Role); - } - [Fact] public void ApplyUpdates_InvalidArgs_Throws() { @@ -125,7 +104,7 @@ public void ApplyUpdates_InvalidArgs_Throws() } [Fact] - public void ApplyUpdates_WithDefaultOptions_UpdatesResponse() + public void ApplyUpdates_UpdatesResponse() { // Arrange var response = new ChatResponse(); @@ -152,50 +131,6 @@ public void ApplyUpdates_WithDefaultOptions_UpdatesResponse() Assert.Equal(new DateTimeOffset(2024, 1, 1, 10, 0, 0, TimeSpan.Zero), message.CreatedAt); } - [Fact] - public void ApplyUpdates_WithOptions_UsesCoalescingOptions() - { - // Arrange - var response = new ChatResponse(); - var existingDataContent = new DataContent("data:text/plain;base64,aGVsbG8=") - { - Name = "shared-name" - }; - response.Messages.Add(new ChatMessage(ChatRole.Assistant, [existingDataContent])); - - var sharedNameDataContent = new DataContent("data:text/plain;base64,dXBkYXRl") - { - Name = "shared-name" // Same name - }; - var otherNameDataContent = new DataContent("data:text/plain;base64,b3RoZXI=") - { - Name = "other-name" // Different name - }; - var updates = new List - { - new() { Contents = [sharedNameDataContent] }, - new() { Contents = [otherNameDataContent] } - }; - - var options = new ChatResponseUpdateCoalescingOptions - { - ReplaceDataContentWithSameName = true - }; - - // Act - response.ApplyUpdates(updates, options); - - // Assert - var message = Assert.Single(response.Messages); - Assert.Equal(2, message.Contents.Count); - - var sharedNameContent = message.Contents.OfType().First(c => c.Name == "shared-name"); - Assert.Equal("data:text/plain;base64,dXBkYXRl", sharedNameContent.Uri); // Should be replaced - - var otherNameContent = message.Contents.OfType().First(c => c.Name == "other-name"); - Assert.Equal("data:text/plain;base64,b3RoZXI=", otherNameContent.Uri); // Should be added - } - [Fact] public void ApplyUpdates_WithEmptyCollection_DoesNothing() { @@ -210,29 +145,6 @@ public void ApplyUpdates_WithEmptyCollection_DoesNothing() Assert.Empty(response.Messages); } - [Fact] - public void ApplyUpdates_WithNullOptions_WorksCorrectly() - { - // Arrange - var response = new ChatResponse(); - var sharedNameDataContent = new DataContent("data:text/plain;base64,dXBkYXRl") - { - Name = "shared-name" // Same name - }; - var updates = new List - { - new(ChatRole.System, [sharedNameDataContent]), - new(ChatRole.System, [sharedNameDataContent]) - }; - - // Act - explicitly pass null options - response.ApplyUpdates(updates, null); - - // Assert - Assert.Single(response.Messages); - Assert.All(response.Messages[0].Contents, c => Assert.IsType(c)); - } - [Fact] public async Task ApplyUpdatesAsync_InvalidArgs_Throws() { @@ -244,7 +156,7 @@ public async Task ApplyUpdatesAsync_InvalidArgs_Throws() } [Fact] - public async Task ApplyUpdatesAsync_WithDefaultOptions_UpdatesResponse() + public async Task ApplyUpdatesAsync_UpdatesResponse() { // Arrange var response = new ChatResponse(); @@ -271,59 +183,6 @@ public async Task ApplyUpdatesAsync_WithDefaultOptions_UpdatesResponse() Assert.Equal(new DateTimeOffset(2024, 1, 1, 11, 0, 0, TimeSpan.Zero), message.CreatedAt); } - [Fact] - public async Task ApplyUpdatesAsync_WithOptions_UsesCoalescingOptions() - { - // Arrange - var response = new ChatResponse(); - var existingDataContent = new DataContent("data:text/plain;base64,b3JpZ2luYWw=") - { - Name = "data-key" - }; - response.Messages.Add(new ChatMessage(ChatRole.Assistant, [existingDataContent])); - - var newDataContent = new DataContent("data:text/plain;base64,bmV3VmFsdWU=") - { - Name = "data-key" // Same name - }; - var updates = new List - { - new() { Contents = [newDataContent] }, - }; - - var options = new ChatResponseUpdateCoalescingOptions - { - ReplaceDataContentWithSameName = true - }; - - // Act - await response.ApplyUpdatesAsync(YieldAsync(updates), options); - - // Assert - var message = Assert.Single(response.Messages); - var dataContent = Assert.Single(message.Contents.OfType()); - Assert.Equal("data:text/plain;base64,bmV3VmFsdWU=", dataContent.Uri); // Should be replaced - Assert.Equal("data-key", dataContent.Name); - } - - [Fact] - public async Task ApplyUpdatesAsync_WithNullOptions_WorksCorrectly() - { - // Arrange - var response = new ChatResponse(); - var updates = new List - { - new(ChatRole.Assistant, "Async message with null options") - }; - - // Act - explicitly pass null options - await response.ApplyUpdatesAsync(YieldAsync(updates), null); - - // Assert - var message = Assert.Single(response.Messages); - Assert.Equal("Async message with null options", message.Text); - } - [Fact] public async Task ApplyUpdatesAsync_MultipleMessages_ProcessedCorrectly() { From ca8b15d9731be071890b343fbd854bc09a75fd60 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 22 Sep 2025 18:50:38 -0700 Subject: [PATCH 09/24] Add ImageGeneratingChatClient --- .../ImageGeneratingChatClient.cs | 194 +++++++++++ ...geGeneratingChatClientBuilderExtensions.cs | 46 +++ .../ImageGeneratingChatClientTests.cs | 303 ++++++++++++++++++ 3 files changed, 543 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs new file mode 100644 index 00000000000..5a248e182f5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -0,0 +1,194 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Tools; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A delegating chat client that enables image generation capabilities by converting instances to function tools. +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// employed is also thread-safe for concurrent use. +/// +/// +/// This client automatically detects instances in the collection +/// and replaces them with equivalent function tools that the chat client can invoke to perform image generation and editing operations. +/// +/// +[Experimental("MEAI001")] +public sealed class ImageGeneratingChatClient : DelegatingChatClient +{ + private readonly IImageGenerator _imageGenerator; + private readonly AITool[] _aiTools; + + /// Stores generated image content from function calls to be included in responses. + private List? _generatedImageContent; + + /// Initializes a new instance of the class. + /// The underlying . + /// An instance that will be used for image generation operations. + /// or is . + public ImageGeneratingChatClient(IChatClient innerClient, IImageGenerator imageGenerator) + : base(innerClient) + { + _imageGenerator = Throw.IfNull(imageGenerator); + _aiTools = + [ + AIFunctionFactory.Create(GenerateImageAsync), + AIFunctionFactory.Create(EditImageAsync) + ]; + } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + // Clear any existing generated content for this request + _generatedImageContent = null; + + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = ProcessChatOptions(options); + + // Get response from base implementation + var response = await base.GetResponseAsync(messages, processedOptions, cancellationToken); + + // If we have generated image content, add it to the response + if (_generatedImageContent is { Count: > 0 }) + { + var lastMessage = response.Messages.LastOrDefault(); + if (lastMessage is not null) + { + // Add generated images to the last message + foreach (var content in _generatedImageContent) + { + lastMessage.Contents.Add(content); + } + } + + // Clear the content after using it + _generatedImageContent = null; + } + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // Clear any existing generated content for this request + _generatedImageContent = null; + + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = ProcessChatOptions(options); + + await foreach (var update in base.GetStreamingResponseAsync(messages, processedOptions, cancellationToken)) + { + // Check if we have generated images since the last update and inject them into this update + if (_generatedImageContent is { Count: > 0 }) + { + // Add generated images to the current update's contents + foreach (var content in _generatedImageContent) + { + update.Contents.Add(content); + } + + // Clear the stored content after using it + _generatedImageContent.Clear(); + } + + yield return update; + } + } + + /// Provides a mechanism for releasing unmanaged resources. + /// to dispose managed resources; otherwise, . + protected override void Dispose(bool disposing) + { + if (disposing) + { + _imageGenerator.Dispose(); + } + + base.Dispose(disposing); + } + + private ChatOptions? ProcessChatOptions(ChatOptions? options) + { + if (options?.Tools is null || options.Tools.Count == 0) + { + return options; + } + + if (!options.Tools.Any(tool => tool is HostedImageGenerationTool)) + { + return options; + } + + var modifiedOptions = options.Clone(); + + // Remove any existing HostedImageGenerationTool instances and add the function tools. + var tools = new List(options.Tools.Count - 1 + _aiTools.Length); + tools.AddRange(options.Tools.Where(tool => tool is not HostedImageGenerationTool)); + tools.AddRange(_aiTools); + + modifiedOptions.Tools = tools; + return modifiedOptions; + } + + [Description("Generates an image based on a text prompt")] + private async Task GenerateImageAsync(string prompt, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + var request = new ImageGenerationRequest(prompt); + var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); + + if (response.Contents.Count == 0) + { + return "No image was generated."; + } + + // Store the generated image content to be included in the response + (_generatedImageContent ??= []).AddRange(response.Contents); + + var imageCount = response.Contents.Count; + return $"Generated {imageCount} image(s) based on the prompt: '{prompt}'"; + } + + [Description("Edits an existing image based on a text prompt and original image data")] + private async Task EditImageAsync(string prompt, string imageData, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + try + { + var imageBytes = Convert.FromBase64String(imageData); + var originalImage = new DataContent(imageBytes, "image/png"); + + var request = new ImageGenerationRequest(prompt, [originalImage]); + var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); + + if (response.Contents.Count == 0) + { + return "No edited image was generated."; + } + + // Store the generated image content to be included in the response + (_generatedImageContent ??= []).AddRange(response.Contents); + + var imageCount = response.Contents.Count; + return $"Edited {imageCount} image(s) based on the prompt: '{prompt}'"; + } + catch (FormatException) + { + return "Invalid image data format. Please provide a valid base64-encoded image."; + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs new file mode 100644 index 00000000000..cf68c824637 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental("MEAI001")] +public static class ImageGeneratingChatClientBuilderExtensions +{ + /// Adds image generation capabilities to the chat client pipeline. + /// The . + /// + /// An optional used for image generation operations. + /// If not supplied, a required instance will be resolved from the service provider. + /// + /// An optional callback that can be used to configure the instance. + /// The . + /// is . + /// + /// + /// This method enables the chat client to handle instances by converting them + /// into function tools that can be invoked by the underlying chat model to perform image generation and editing operations. + /// + /// + public static ChatClientBuilder UseImageGeneration( + this ChatClientBuilder builder, + IImageGenerator? imageGenerator = null, + Action? configure = null) + { + _ = Throw.IfNull(builder); + + return builder.Use((innerClient, services) => + { + imageGenerator ??= services.GetRequiredService(); + + var chatClient = new ImageGeneratingChatClient(innerClient, imageGenerator); + configure?.Invoke(chatClient); + return chatClient; + }); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs new file mode 100644 index 00000000000..37aeead2e90 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Tools; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class ImageGeneratingChatClientTests +{ + [Fact] + public void ImageGeneratingChatClient_InvalidArgs_Throws() + { + using var innerClient = new TestChatClient(); + using var imageGenerator = new TestImageGenerator(); + + Assert.Throws("innerClient", () => new ImageGeneratingChatClient(null!, imageGenerator)); + Assert.Throws("imageGenerator", () => new ImageGeneratingChatClient(innerClient, null!)); + } + + [Fact] + public void UseImageGeneration_WithNullBuilder_Throws() + { + Assert.Throws("builder", () => ((ChatClientBuilder)null!).UseImageGeneration()); + } + + [Fact] + public async Task GetResponseAsync_WithoutImageGenerationTool_PassesThrough() + { + // Arrange + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [AIFunctionFactory.Create(() => "dummy function", name: "DummyFunction")] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.NotNull(response); + Assert.Equal("test response", response.Messages[0].Text); + + // Verify that tools collection still has the original function, not replaced + Assert.Single(chatOptions.Tools); + Assert.IsAssignableFrom(chatOptions.Tools[0]); + } + + [Fact] + public async Task GetResponseAsync_WithImageGenerationTool_ReplacesTool() + { + // Arrange + bool innerClientCalled = false; + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + innerClientCalled = true; + capturedOptions = options; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.True(innerClientCalled); + Assert.NotNull(capturedOptions); + Assert.NotNull(capturedOptions.Tools); + Assert.Equal(2, capturedOptions.Tools.Count); // Should have GenerateImage and EditImage functions + + // Verify the functions are properly created + var generateImageFunction = capturedOptions.Tools[0] as AIFunction; + var editImageFunction = capturedOptions.Tools[1] as AIFunction; + + Assert.NotNull(generateImageFunction); + Assert.NotNull(editImageFunction); + Assert.Equal("GenerateImage", generateImageFunction.Name); + Assert.Equal("EditImage", editImageFunction.Name); + } + + [Fact] + public async Task GetResponseAsync_WithMixedTools_ReplacesOnlyImageGenerationTool() + { + // Arrange + bool innerClientCalled = false; + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + innerClientCalled = true; + capturedOptions = options; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var dummyFunction = AIFunctionFactory.Create(() => "dummy", name: "DummyFunction"); + var chatOptions = new ChatOptions + { + Tools = [dummyFunction, new HostedImageGenerationTool()] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.True(innerClientCalled); + Assert.NotNull(capturedOptions); + Assert.NotNull(capturedOptions.Tools); + Assert.Equal(3, capturedOptions.Tools.Count); // DummyFunction + GenerateImage + EditImage + + Assert.Same(dummyFunction, capturedOptions.Tools[0]); // Original function preserved + Assert.IsAssignableFrom(capturedOptions.Tools[1]); // GenerateImage function + Assert.IsAssignableFrom(capturedOptions.Tools[2]); // EditImage function + } + + [Fact] + public void UseImageGeneration_ServiceProviderIntegration_Works() + { + // Arrange + var services = new ServiceCollection(); + services.AddSingleton(); + + using var serviceProvider = services.BuildServiceProvider(); + using var innerClient = new TestChatClient(); + + // Act + using var client = innerClient + .AsBuilder() + .UseImageGeneration() + .Build(serviceProvider); + + // Assert + Assert.IsType(client); + } + + [Fact] + public void UseImageGeneration_WithProvidedImageGenerator_Works() + { + // Arrange + using var innerClient = new TestChatClient(); + using var imageGenerator = new TestImageGenerator(); + + // Act + using var client = innerClient + .AsBuilder() + .UseImageGeneration(imageGenerator) + .Build(); + + // Assert + Assert.IsType(client); + } + + [Fact] + public void UseImageGeneration_WithConfigureCallback_CallsCallback() + { + // Arrange + using var innerClient = new TestChatClient(); + using var imageGenerator = new TestImageGenerator(); + bool configureCallbackInvoked = false; + + // Act + using var client = innerClient + .AsBuilder() + .UseImageGeneration(imageGenerator, configure: c => + { + Assert.NotNull(c); + configureCallbackInvoked = true; + }) + .Build(); + + // Assert + Assert.True(configureCallbackInvoked); + } + + [Fact] + public async Task GetStreamingResponseAsync_WithImageGenerationTool_ReplacesTool() + { + // Arrange + bool innerClientCalled = false; + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetStreamingResponseAsyncCallback = (messages, options, cancellationToken) => + { + innerClientCalled = true; + capturedOptions = options; + return GetUpdatesAsync(); + } + }; + + static async IAsyncEnumerable GetUpdatesAsync() + { + await Task.Yield(); + yield return new(ChatRole.Assistant, "test"); + } + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + await foreach (var update in client.GetStreamingResponseAsync([new(ChatRole.User, "test")], chatOptions)) + { + // Process updates + } + + // Assert + Assert.True(innerClientCalled); + Assert.NotNull(capturedOptions); + Assert.NotNull(capturedOptions.Tools); + Assert.Equal(2, capturedOptions.Tools.Count); // GenerateImage and EditImage functions + } + + [Fact] + public async Task GetResponseAsync_WithNullOptions_DoesNotThrow() + { + // Arrange + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + // Act & Assert + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], null); + Assert.NotNull(response); + } + + [Fact] + public async Task GetResponseAsync_WithEmptyTools_DoesNotModify() + { + // Arrange + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + capturedOptions = options; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [] + }; + + // Act + await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.Same(chatOptions, capturedOptions); +#pragma warning disable CA1508 + Assert.NotNull(capturedOptions?.Tools); +#pragma warning restore CA1508 + Assert.Empty(capturedOptions.Tools); + } +} From 62e0ac5520167278d4b0d8fe7fbf08c7ae358424 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 26 Sep 2025 09:30:46 -0700 Subject: [PATCH 10/24] Fix namespace of tool --- .../Tools/HostedImageGenerationTool.cs | 2 +- .../Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs | 1 - .../ChatCompletion/ImageGeneratingChatClient.cs | 1 - .../ImageGeneratingChatClientBuilderExtensions.cs | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs index 897fd4f9f89..fcf83818e74 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs @@ -3,7 +3,7 @@ using System.Diagnostics.CodeAnalysis; -namespace Microsoft.Extensions.AI.Tools; +namespace Microsoft.Extensions.AI; /// Represents a hosted tool that can be specified to an AI service to enable it to perform image generation. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 452d63a182a..4d8e7a05696 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -13,7 +13,6 @@ using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Tools; using Microsoft.Shared.Diagnostics; using OpenAI.Images; using OpenAI.Responses; diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index 5a248e182f5..f7513641758 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -9,7 +9,6 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Tools; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs index cf68c824637..241c851fd4e 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs @@ -23,7 +23,7 @@ public static class ImageGeneratingChatClientBuilderExtensions /// is . /// /// - /// This method enables the chat client to handle instances by converting them + /// This method enables the chat client to handle instances by converting them /// into function tools that can be invoked by the underlying chat model to perform image generation and editing operations. /// /// From 81e6e5afbea750f75a8260b73705a45bbe7ae047 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 26 Sep 2025 09:31:37 -0700 Subject: [PATCH 11/24] Replace traces of function calling --- .../ImageGeneratingChatClient.cs | 213 +++++++++++++----- 1 file changed, 162 insertions(+), 51 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index f7513641758..a857abe38a8 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -29,9 +29,11 @@ public sealed class ImageGeneratingChatClient : DelegatingChatClient { private readonly IImageGenerator _imageGenerator; private readonly AITool[] _aiTools; + private readonly HashSet _functionNames; - /// Stores generated image content from function calls to be included in responses. - private List? _generatedImageContent; + /// Stores mapping of function call IDs to generated image content. + private readonly Dictionary> _imageContentByCallId = []; + private ImageGenerationOptions? _imageGenerationOptions; /// Initializes a new instance of the class. /// The underlying . @@ -46,6 +48,8 @@ public ImageGeneratingChatClient(IChatClient innerClient, IImageGenerator imageG AIFunctionFactory.Create(GenerateImageAsync), AIFunctionFactory.Create(EditImageAsync) ]; + + _functionNames = new(_aiTools.Select(t => t.Name), StringComparer.Ordinal); } /// @@ -53,7 +57,7 @@ public override async Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { // Clear any existing generated content for this request - _generatedImageContent = null; + _imageContentByCallId.Clear(); // Process the chat options to replace HostedImageGenerationTool with functions var processedOptions = ProcessChatOptions(options); @@ -61,21 +65,11 @@ public override async Task GetResponseAsync( // Get response from base implementation var response = await base.GetResponseAsync(messages, processedOptions, cancellationToken); - // If we have generated image content, add it to the response - if (_generatedImageContent is { Count: > 0 }) + // Replace FunctionResultContent instances with generated image content + foreach (var message in response.Messages) { - var lastMessage = response.Messages.LastOrDefault(); - if (lastMessage is not null) - { - // Add generated images to the last message - foreach (var content in _generatedImageContent) - { - lastMessage.Contents.Add(content); - } - } - - // Clear the content after using it - _generatedImageContent = null; + var newContents = ReplaceImageGenerationFunctionResults(message.Contents); + message.Contents = newContents; } return response; @@ -86,27 +80,35 @@ public override async IAsyncEnumerable GetStreamingResponseA IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { // Clear any existing generated content for this request - _generatedImageContent = null; + _imageContentByCallId.Clear(); // Process the chat options to replace HostedImageGenerationTool with functions var processedOptions = ProcessChatOptions(options); await foreach (var update in base.GetStreamingResponseAsync(messages, processedOptions, cancellationToken)) { - // Check if we have generated images since the last update and inject them into this update - if (_generatedImageContent is { Count: > 0 }) + // Replace any FunctionResultContent instances with generated image content + var newContents = ReplaceImageGenerationFunctionResults(update.Contents); + + if (newContents != update.Contents) { - // Add generated images to the current update's contents - foreach (var content in _generatedImageContent) + // Create a new update instance with modified contents + var modifiedUpdate = new ChatResponseUpdate(update.Role, newContents) { - update.Contents.Add(content); - } - - // Clear the stored content after using it - _generatedImageContent.Clear(); + AuthorName = update.AuthorName, + RawRepresentation = update.RawRepresentation, + AdditionalProperties = update.AdditionalProperties, + ResponseId = update.ResponseId, + MessageId = update.MessageId, + ConversationId = update.ConversationId + }; + + yield return modifiedUpdate; + } + else + { + yield return update; } - - yield return update; } } @@ -129,58 +131,166 @@ protected override void Dispose(bool disposing) return options; } - if (!options.Tools.Any(tool => tool is HostedImageGenerationTool)) + var tools = options.Tools; + ChatOptions? modifiedOptions = null; + + for (int i = 0; i < tools.Count; i++) { - return options; - } + var tool = options.Tools[i]; - var modifiedOptions = options.Clone(); + // remove all instances of HostedImageGenerationTool and store the options from the last one + if (tool is HostedImageGenerationTool imageGenerationTool) + { + _imageGenerationOptions = imageGenerationTool.Options; + +#pragma warning disable S127 + // for the first image generation tool, clone the options and insert our function tools + // remove any subsequent image generation tools + if (modifiedOptions is null) + { + modifiedOptions = options.Clone(); + tools = modifiedOptions.Tools!; - // Remove any existing HostedImageGenerationTool instances and add the function tools. - var tools = new List(options.Tools.Count - 1 + _aiTools.Length); - tools.AddRange(options.Tools.Where(tool => tool is not HostedImageGenerationTool)); - tools.AddRange(_aiTools); + tools.RemoveAt(i--); - modifiedOptions.Tools = tools; - return modifiedOptions; + foreach (var functionTool in _aiTools) + { + tools.Insert(++i, functionTool); + } + } + else + { + tools.RemoveAt(i--); + } +#pragma warning restore S127 + } + } + + return modifiedOptions ?? options; } - [Description("Generates an image based on a text prompt")] - private async Task GenerateImageAsync(string prompt, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. + /// The list of AI content to process. + private IList ReplaceImageGenerationFunctionResults(IList contents) + { + IList? newContents = null; + +#pragma warning disable S127 +#pragma warning disable S125 + // Replace FunctionResultContent instances with generated image content + for (int i = contents.Count - 1; i >= 0; i--) + { + var content = contents[i]; + + if (content is FunctionCallContent functionCall && + _functionNames.Contains(functionCall.Name)) + { + EnsureNewContents(); + contents.RemoveAt(i--); + } + + if (content is FunctionResultContent functionResult && + _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) + { + // Remove the function result + EnsureNewContents(); + contents.RemoveAt(i); + + // Insert generated image content in its place + for (int j = imageContents.Count - 1; j >= 0; j--) + { + contents.Insert(i, imageContents[j]); + } + + _ = _imageContentByCallId.Remove(functionResult.CallId); + } + } + + return contents; + + void EnsureNewContents() + { + if (newContents is null) + { + newContents = [.. contents]; + contents = newContents; + } + } + } +#pragma warning disable EA0014 + [Description("Generates images based on a text description")] + private async Task GenerateImageAsync( + [Description("A detailed description of the image to generate")] string prompt) { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return "No call ID available for image generation."; + } + var request = new ImageGenerationRequest(prompt); - var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); + var options = _imageGenerationOptions ?? new ImageGenerationOptions(); + options.Count ??= 1; + + var response = await _imageGenerator.GenerateAsync(request, options); if (response.Contents.Count == 0) { return "No image was generated."; } - // Store the generated image content to be included in the response - (_generatedImageContent ??= []).AddRange(response.Contents); + // Store the generated image content mapped to this call ID + _imageContentByCallId[callId] = [.. response.Contents]; + + int imageCount = 0; + List imageIds = []; + + foreach (var content in response.Contents) + { + if (content is DataContent imageContent) + { + imageCount++; - var imageCount = response.Contents.Count; - return $"Generated {imageCount} image(s) based on the prompt: '{prompt}'"; + // if there is no name, generate one based on the call ID and index + imageContent.Name ??= $"{callId}_image_{imageCount}"; + imageIds.Add(imageContent.Name); + + imageContent.AdditionalProperties ??= new(); + imageContent.AdditionalProperties["prompt"] = prompt; + } + } + + return $"Generated {imageCount} image(s) with IDs: {string.Join(",", imageIds)} based on the prompt: '{prompt}'"; } - [Description("Edits an existing image based on a text prompt and original image data")] - private async Task EditImageAsync(string prompt, string imageData, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + [Description("Edits an existing image based on a text description")] + private async Task EditImageAsync( + [Description("A detailed description of the image to generate")] string prompt, + [Description("The original image content to edit")] string imageData) { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return "No call ID available for image editing."; + } + try { var imageBytes = Convert.FromBase64String(imageData); var originalImage = new DataContent(imageBytes, "image/png"); var request = new ImageGenerationRequest(prompt, [originalImage]); - var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); + var response = await _imageGenerator.GenerateAsync(request, _imageGenerationOptions); if (response.Contents.Count == 0) { return "No edited image was generated."; } - // Store the generated image content to be included in the response - (_generatedImageContent ??= []).AddRange(response.Contents); + // Store the generated image content mapped to this call ID + _imageContentByCallId[callId] = [.. response.Contents]; var imageCount = response.Contents.Count; return $"Edited {imageCount} image(s) based on the prompt: '{prompt}'"; @@ -190,4 +300,5 @@ private async Task EditImageAsync(string prompt, string imageData, Image return "Invalid image data format. Please provide a valid base64-encoded image."; } } +#pragma warning restore EA0014 } From 6559a66106c840a70e3eea82b5cb51165cd984f1 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 26 Sep 2025 15:36:04 -0700 Subject: [PATCH 12/24] More namepsace fix --- .../ChatCompletion/ImageGeneratingChatClientTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs index 37aeead2e90..b8e046d7263 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Tools; +using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using Xunit; From 398bbdb94a24edef095e9a499fca5533d7ea6552 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 29 Sep 2025 18:17:44 -0700 Subject: [PATCH 13/24] Enable editing --- .../ImageGeneratingChatClient.cs | 375 +++++++++++++----- 1 file changed, 270 insertions(+), 105 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index a857abe38a8..e9c372c6ab6 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -27,89 +27,157 @@ namespace Microsoft.Extensions.AI; [Experimental("MEAI001")] public sealed class ImageGeneratingChatClient : DelegatingChatClient { + /// + /// Specifies how image and other data content is handled when passing data to an inner client. + /// + /// + /// Use this enumeration to control whether images in the data content are passed as-is, replaced + /// with unique identifiers, or only generated images are replaced. This setting affects how downstream clients + /// receive and process image data. + /// Reducing what's passed downstream can help manage the context window. + /// + public enum DataContentHandling + { + /// Pass all DataContent to inner client. + None, + + /// Replace all images with unique identifers when passing to inner client. + AllImages, + + /// Replace only images that were produced by past of image generation requests with unique identifiers when passing to inner client. + GeneratedImages + } + + private const string ImageKey = "meai_image"; + private readonly IImageGenerator _imageGenerator; private readonly AITool[] _aiTools; private readonly HashSet _functionNames; + private readonly DataContentHandling _dataContentHandling; - /// Stores mapping of function call IDs to generated image content. + // the following fields all have scope per-request. They are cleared at the start of each request. private readonly Dictionary> _imageContentByCallId = []; + private readonly Dictionary _imageContentById = new(StringComparer.OrdinalIgnoreCase); private ImageGenerationOptions? _imageGenerationOptions; + private static List CopyList(IList original, int toOffsetExclusive, int additionalCapacity = 0) + { + var newList = new List(original.Count + additionalCapacity); + + // Copy all items up to and excluding the current index + for (int j = 0; j < toOffsetExclusive; j++) + { + newList.Add(original[j]); + } + + return newList; + } + /// Initializes a new instance of the class. /// The underlying . /// An instance that will be used for image generation operations. + /// Specifies how to handle instances when passing messages to the inner client. + /// The default is . /// or is . - public ImageGeneratingChatClient(IChatClient innerClient, IImageGenerator imageGenerator) + public ImageGeneratingChatClient(IChatClient innerClient, IImageGenerator imageGenerator, DataContentHandling dataContentHandling = DataContentHandling.AllImages) : base(innerClient) { _imageGenerator = Throw.IfNull(imageGenerator); _aiTools = [ AIFunctionFactory.Create(GenerateImageAsync), - AIFunctionFactory.Create(EditImageAsync) + AIFunctionFactory.Create(EditImageAsync), + AIFunctionFactory.Create(GetImagesForEdit) ]; _functionNames = new(_aiTools.Select(t => t.Name), StringComparer.Ordinal); + _dataContentHandling = dataContentHandling; } /// public override async Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { + _ = Throw.IfNull(messages); + // Clear any existing generated content for this request _imageContentByCallId.Clear(); + _imageContentById.Clear(); + + try + { + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = ProcessChatOptions(options); + var processedMessages = ProcessChatMessages(messages); - // Process the chat options to replace HostedImageGenerationTool with functions - var processedOptions = ProcessChatOptions(options); + // Get response from base implementation + var response = await base.GetResponseAsync(processedMessages, processedOptions, cancellationToken); - // Get response from base implementation - var response = await base.GetResponseAsync(messages, processedOptions, cancellationToken); + // Replace FunctionResultContent instances with generated image content + foreach (var message in response.Messages) + { + var newContents = ReplaceImageGenerationFunctionResults(message.Contents); + message.Contents = newContents; + } - // Replace FunctionResultContent instances with generated image content - foreach (var message in response.Messages) + return response; + } + finally { - var newContents = ReplaceImageGenerationFunctionResults(message.Contents); - message.Contents = newContents; + // Clear any existing generated content for this request + _imageContentByCallId.Clear(); + _imageContentById.Clear(); } - - return response; } /// public override async IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + _ = Throw.IfNull(messages); + // Clear any existing generated content for this request _imageContentByCallId.Clear(); + _imageContentById.Clear(); - // Process the chat options to replace HostedImageGenerationTool with functions - var processedOptions = ProcessChatOptions(options); - - await foreach (var update in base.GetStreamingResponseAsync(messages, processedOptions, cancellationToken)) + try { - // Replace any FunctionResultContent instances with generated image content - var newContents = ReplaceImageGenerationFunctionResults(update.Contents); + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = ProcessChatOptions(options); + var processedMessages = ProcessChatMessages(messages); - if (newContents != update.Contents) + await foreach (var update in base.GetStreamingResponseAsync(processedMessages, processedOptions, cancellationToken)) { - // Create a new update instance with modified contents - var modifiedUpdate = new ChatResponseUpdate(update.Role, newContents) + // Replace any FunctionResultContent instances with generated image content + var newContents = ReplaceImageGenerationFunctionResults(update.Contents); + + if (newContents != update.Contents) { - AuthorName = update.AuthorName, - RawRepresentation = update.RawRepresentation, - AdditionalProperties = update.AdditionalProperties, - ResponseId = update.ResponseId, - MessageId = update.MessageId, - ConversationId = update.ConversationId - }; - - yield return modifiedUpdate; - } - else - { - yield return update; + // Create a new update instance with modified contents + var modifiedUpdate = new ChatResponseUpdate(update.Role, newContents) + { + AuthorName = update.AuthorName, + RawRepresentation = update.RawRepresentation, + AdditionalProperties = update.AdditionalProperties, + ResponseId = update.ResponseId, + MessageId = update.MessageId, + ConversationId = update.ConversationId + }; + + yield return modifiedUpdate; + } + else + { + yield return update; + } } } + finally + { + // Clear any existing generated content for this request + _imageContentByCallId.Clear(); + _imageContentById.Clear(); + } } /// Provides a mechanism for releasing unmanaged resources. @@ -124,6 +192,71 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } + private IEnumerable ProcessChatMessages(IEnumerable messages) + { + // If no special handling is needed, return the original messages + if (_dataContentHandling == DataContentHandling.None) + { + return messages; + } + + List? newMessages = null; + int messageIndex = 0; + foreach (var message in messages) + { + List? newContents = null; + for (int contentIndex = 0; contentIndex < message.Contents.Count; contentIndex++) + { + var content = message.Contents[contentIndex]; + if (content is DataContent dataContent && dataContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + bool isGeneratedImage = dataContent.AdditionalProperties?.ContainsKey(ImageKey) == true; + if (_dataContentHandling == DataContentHandling.AllImages || + (_dataContentHandling == DataContentHandling.GeneratedImages && isGeneratedImage)) + { + // Replace image with a placeholder text content + var imageId = StoreImage(dataContent); + + newContents ??= CopyList(message.Contents, contentIndex); + newContents.Add(new TextContent($"[{ImageKey}:{imageId}] available for edit.") + { + Annotations = dataContent.Annotations, + AdditionalProperties = dataContent.AdditionalProperties + }); + continue; // Skip adding the original content + } + } + + // Add the original content if no replacement was made + newContents?.Add(content); + } + + if (newContents != null) + { + newMessages ??= new List(messages.Take(messageIndex)); + + var newMessage = message.Clone(); + if (newMessage.Role == ChatRole.Tool) + { + // workaround: the chat client will ignore tool messages, so change the role to assistant + newMessage.Role = ChatRole.Assistant; + } + + newMessage.Contents = newContents; + newMessages.Add(newMessage); + } + else + { + newMessages?.Add(message); + + } + + messageIndex++; + } + + return newMessages ?? messages; + } + private ChatOptions? ProcessChatOptions(ChatOptions? options) { if (options?.Tools is null || options.Tools.Count == 0) @@ -131,52 +264,78 @@ protected override void Dispose(bool disposing) return options; } + List? newTools = null; var tools = options.Tools; - ChatOptions? modifiedOptions = null; - for (int i = 0; i < tools.Count; i++) { - var tool = options.Tools[i]; + var tool = tools[i]; // remove all instances of HostedImageGenerationTool and store the options from the last one if (tool is HostedImageGenerationTool imageGenerationTool) { _imageGenerationOptions = imageGenerationTool.Options; -#pragma warning disable S127 // for the first image generation tool, clone the options and insert our function tools // remove any subsequent image generation tools - if (modifiedOptions is null) + if (newTools is null) { - modifiedOptions = options.Clone(); - tools = modifiedOptions.Tools!; - - tools.RemoveAt(i--); - - foreach (var functionTool in _aiTools) - { - tools.Insert(++i, functionTool); - } + newTools = CopyList(tools, i); + newTools.AddRange(_aiTools); } - else - { - tools.RemoveAt(i--); - } -#pragma warning restore S127 } + else + { + newTools?.Add(tool); + } + } + + if (newTools is not null) + { + var newOptions = options.Clone(); + newOptions.Tools = newTools; + return newOptions; } - return modifiedOptions ?? options; + return options; + } + + private DataContent? RetrieveImageContent(string imageId) + { + if (_imageContentById.TryGetValue(imageId, out var imageContent)) + { + return imageContent as DataContent; + } + + return null; + } + + private string StoreImage(DataContent imageContent, bool isGenerated = false) + { + // Generate a unique ID for the image if it doesn't have one + string? imageId = null; + if (imageContent.AdditionalProperties?.TryGetValue(ImageKey, out imageId) is false || imageId is null) + { + imageId = imageContent.Name ?? Guid.NewGuid().ToString(); + } + + if (isGenerated) + { + imageContent.AdditionalProperties ??= new(); + imageContent.AdditionalProperties[ImageKey] = imageId; + } + + // Store the image content for later retrieval + _imageContentById[imageId] = imageContent; + + return imageId; } /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. /// The list of AI content to process. private IList ReplaceImageGenerationFunctionResults(IList contents) { - IList? newContents = null; + List? newContents = null; -#pragma warning disable S127 -#pragma warning disable S125 // Replace FunctionResultContent instances with generated image content for (int i = contents.Count - 1; i >= 0; i--) { @@ -185,42 +344,37 @@ private IList ReplaceImageGenerationFunctionResults(IList if (content is FunctionCallContent functionCall && _functionNames.Contains(functionCall.Name)) { - EnsureNewContents(); - contents.RemoveAt(i--); + // create a new list and skip this + newContents ??= CopyList(contents, i); } - - if (content is FunctionResultContent functionResult && + else if (content is FunctionResultContent functionResult && _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) { - // Remove the function result - EnsureNewContents(); - contents.RemoveAt(i); + newContents ??= CopyList(contents, i, imageContents.Count - 1); // Insert generated image content in its place - for (int j = imageContents.Count - 1; j >= 0; j--) + foreach (var imageContent in imageContents) { - contents.Insert(i, imageContents[j]); + newContents.Add(imageContent); } + // Remove the mapping as it's no longer needed _ = _imageContentByCallId.Remove(functionResult.CallId); } - } - - return contents; - - void EnsureNewContents() - { - if (newContents is null) + else { - newContents = [.. contents]; - contents = newContents; + // keep the existing content if we have a new list + newContents?.Add(content); } } + + return newContents ?? contents; } -#pragma warning disable EA0014 - [Description("Generates images based on a text description")] + + [Description("Generates images based on a text description.")] private async Task GenerateImageAsync( - [Description("A detailed description of the image to generate")] string prompt) + [Description("A detailed description of the image to generate")] string prompt, + CancellationToken cancellationToken = default) { // Get the call ID from the current function invocation context var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; @@ -233,41 +387,38 @@ private async Task GenerateImageAsync( var options = _imageGenerationOptions ?? new ImageGenerationOptions(); options.Count ??= 1; - var response = await _imageGenerator.GenerateAsync(request, options); + var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); if (response.Contents.Count == 0) { return "No image was generated."; } - // Store the generated image content mapped to this call ID - _imageContentByCallId[callId] = [.. response.Contents]; - - int imageCount = 0; List imageIds = []; - + List imageContents = _imageContentByCallId[callId] = []; foreach (var content in response.Contents) { - if (content is DataContent imageContent) + if (content is DataContent imageContent && imageContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) { - imageCount++; - - // if there is no name, generate one based on the call ID and index - imageContent.Name ??= $"{callId}_image_{imageCount}"; - imageIds.Add(imageContent.Name); - - imageContent.AdditionalProperties ??= new(); - imageContent.AdditionalProperties["prompt"] = prompt; + imageContents.Add(imageContent); + imageIds.Add(StoreImage(imageContent, true)); } } - return $"Generated {imageCount} image(s) with IDs: {string.Join(",", imageIds)} based on the prompt: '{prompt}'"; + return "Generated image successfully."; } - [Description("Edits an existing image based on a text description")] + [Description("Lists the identifiers of all images available for edit.")] + private string[] GetImagesForEdit() + { + return _imageContentById.Keys.ToArray(); + } + + [Description("Edits an existing image based on a text description.")] private async Task EditImageAsync( [Description("A detailed description of the image to generate")] string prompt, - [Description("The original image content to edit")] string imageData) + [Description($"The image to edit from one of the available image identifiers returned by {nameof(GetImagesForEdit)}")] string imageId, + CancellationToken cancellationToken = default) { // Get the call ID from the current function invocation context var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; @@ -276,29 +427,43 @@ private async Task EditImageAsync( return "No call ID available for image editing."; } + if (string.IsNullOrEmpty(imageId)) + { + return "No imageId provided"; + } + try { - var imageBytes = Convert.FromBase64String(imageData); - var originalImage = new DataContent(imageBytes, "image/png"); + var originalImage = RetrieveImageContent(imageId); + if (originalImage == null) + { + return $"No image found with: {imageId}"; + } var request = new ImageGenerationRequest(prompt, [originalImage]); - var response = await _imageGenerator.GenerateAsync(request, _imageGenerationOptions); + var response = await _imageGenerator.GenerateAsync(request, _imageGenerationOptions, cancellationToken); if (response.Contents.Count == 0) { return "No edited image was generated."; } - // Store the generated image content mapped to this call ID - _imageContentByCallId[callId] = [.. response.Contents]; + List imageIds = []; + List imageContents = _imageContentByCallId[callId] = []; + foreach (var content in response.Contents) + { + if (content is DataContent imageContent && imageContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + imageContents.Add(imageContent); + imageIds.Add(StoreImage(imageContent, true)); + } + } - var imageCount = response.Contents.Count; - return $"Edited {imageCount} image(s) based on the prompt: '{prompt}'"; + return "Edited image successfully."; } catch (FormatException) { return "Invalid image data format. Please provide a valid base64-encoded image."; } } -#pragma warning restore EA0014 } From 1d96532d695ecdfec2f290c1bdd094623a9f468a Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 30 Sep 2025 19:09:49 -0700 Subject: [PATCH 14/24] Update to preview OpenAI with image tool support --- NuGet.config | 5 + eng/packages/General.props | 2 +- .../Tools/HostedImageGenerationTool.cs | 21 +++ .../Microsoft.Extensions.AI.OpenAI.csproj | 1 - .../OpenAIResponsesChatClient.cs | 149 +++++------------- 5 files changed, 67 insertions(+), 111 deletions(-) diff --git a/NuGet.config b/NuGet.config index 0fedd015e82..a8e7ecd2c9c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -17,6 +17,8 @@ + + @@ -39,6 +41,9 @@ + + + diff --git a/eng/packages/General.props b/eng/packages/General.props index 5be4031ad4d..b9f4b4c90ae 100644 --- a/eng/packages/General.props +++ b/eng/packages/General.props @@ -16,7 +16,7 @@ - + diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs index fcf83818e74..e16cb1ef094 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -25,4 +27,23 @@ public HostedImageGenerationTool() /// Gets or sets the options used to configure image generation. /// public ImageGenerationOptions? Options { get; set; } + + /// + /// Gets or sets a callback responsible for creating the raw representation of the image generation tool from an underlying implementation. + /// + /// + /// The underlying implementation can have its own representation of this tool. + /// When or is invoked with an + /// , that implementation can convert the provided tool and options into its own representation in + /// order to use it while performing the operation. For situations where a consumer knows which concrete is being used + /// and how it represents this tool, a new instance of that implementation-specific tool type can be returned by this + /// callback for the implementation to use instead of creating a new instance. + /// Such implementations might mutate the supplied options instance further based on other settings supplied on this + /// instance or from other inputs, therefore, it is strongly recommended to not + /// return shared instances and instead make the callback return a new instance on each call. + /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly typed + /// properties on . + /// + [JsonIgnore] + public Func? RawRepresentationFactory { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj index 666db46bfc1..216fca4d272 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj @@ -27,7 +27,6 @@ true true true - true true diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 1388d1b0fb7..798fe1c3a90 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -14,7 +14,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; -using OpenAI.Images; using OpenAI.Responses; #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields @@ -201,16 +200,12 @@ internal static IEnumerable ToChatMessages(IEnumerable ToChatMessages(IEnumerable? additionalRawData = imageGenResultType + IDictionary? additionalRawData = typeof(ResponseItem) .GetProperty("SerializedAdditionalRawData", InternalBindingFlags) ?.GetValue(outputItem) as IDictionary; - // Properties - // background - // output_format - // quality - // revised_prompt - // size - string outputFormat = getStringProperty("output_format") ?? "png"; - var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty); - - return new DataContent(resultBytes, $"image/{outputFormat}") + return new DataContent(outputItem.GeneratedImageBytes, $"image/{outputFormat}") { - RawRepresentation = outputItem, - AdditionalProperties = new() - { - ["background"] = getStringProperty("background"), - ["output_format"] = outputFormat, - ["quality"] = getStringProperty("quality"), - ["revised_prompt"] = getStringProperty("revised_prompt"), - ["size"] = getStringProperty("size"), - ["status"] = imageGenStatus, - } + RawRepresentation = outputItem }; string? getStringProperty(string name) @@ -276,21 +244,11 @@ private static DataContent GetContentFromImageGen(ResponseItem outputItem) } [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(ResponseItem))] - private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResponseUpdate update) + private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResponseImageGenerationCallPartialImageUpdate update) { const BindingFlags InternalBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; - var partialImageEventType = Type.GetType("OpenAI.Responses.InternalResponseImageGenCallPartialImageEvent, OpenAI"); - if (partialImageEventType == null) - { - throw new InvalidOperationException("Unable to determine the type of the image generation result."); - } - - var imageGenResult = partialImageEventType.GetProperty("PartialImageB64", InternalBindingFlags)?.GetValue(update) as string; - var imageGenItemId = partialImageEventType.GetProperty("ItemId", InternalBindingFlags)?.GetValue(update) as string; - var imageGenOutputIndex = partialImageEventType.GetProperty("OutputIndex", InternalBindingFlags)?.GetValue(update) as int?; - var imageGenPartialImageIndex = partialImageEventType.GetProperty("PartialImageIndex", InternalBindingFlags)?.GetValue(update) as int?; - IDictionary? additionalRawData = partialImageEventType + IDictionary? additionalRawData = typeof(ResponseItem) .GetProperty("SerializedAdditionalRawData", InternalBindingFlags) ?.GetValue(update) as IDictionary; @@ -303,21 +261,14 @@ private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResp string outputFormat = getStringProperty("output_format") ?? "png"; - var resultBytes = Convert.FromBase64String(imageGenResult ?? string.Empty); - - return new DataContent(resultBytes, $"image/{outputFormat}") + return new DataContent(update.PartialImageBytes, $"image/{outputFormat}") { RawRepresentation = update, AdditionalProperties = new() { - ["ItemId"] = imageGenItemId, - ["OutputIndex"] = imageGenOutputIndex, - ["PartialImageIndex"] = imageGenPartialImageIndex, - ["background"] = getStringProperty("background"), - ["output_format"] = outputFormat, - ["quality"] = getStringProperty("quality"), - ["revised_prompt"] = getStringProperty("revised_prompt"), - ["size"] = getStringProperty("size"), + ["ItemId"] = update.ItemId, + ["OutputIndex"] = update.OutputIndex, + ["PartialImageIndex"] = update.PartialImageIndex } }; @@ -474,17 +425,12 @@ outputItemDoneUpdate.Item is MessageResponseItem mri && yield return CreateUpdate(new TextReasoningContent(delta)); break; - default: - - if (streamingUpdate.GetType().Name == "InternalResponseImageGenCallPartialImageEvent") - { - yield return CreateUpdate(GetContentFromImageGenPartialImageEvent(streamingUpdate)); - } - else - { - yield return CreateUpdate(); - } + case StreamingResponseImageGenerationCallPartialImageUpdate streamingImageGenUpdate: + yield return CreateUpdate(GetContentFromImageGenPartialImageEvent(streamingImageGenUpdate)); + break; + default: + yield return CreateUpdate(); break; } } @@ -509,57 +455,42 @@ internal static FunctionTool ToResponseTool(AIFunctionDeclaration aiFunction, Ch aiFunction.Description); } - internal static ResponseTool ToImageResponseTool(HostedImageGenerationTool imageGenerationTool, ChatOptions? options = null) + internal ResponseTool ToImageResponseTool(HostedImageGenerationTool imageGenerationTool) { - ImageGenerationOptions? imageGenerationOptions = null; - if (imageGenerationTool.AdditionalProperties.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj)) - { - imageGenerationOptions = optionsObj as ImageGenerationOptions; - } - else if (options?.AdditionalProperties?.TryGetValue(nameof(ImageGenerationOptions), out object? optionsObj2) ?? false) + ImageGenerationOptions? imageGenerationOptions = imageGenerationTool.Options; + + // A bit unusual to get an ImageGenerationTool from the ImageGenerationOptions factory, we could + var result = imageGenerationTool.RawRepresentationFactory?.Invoke(this) as ImageGenerationTool ?? new(); + + // Model: Image generation model + if (imageGenerationOptions?.ModelId is not null && result.Model is null) { - imageGenerationOptions = optionsObj2 as ImageGenerationOptions; + result.Model = imageGenerationOptions.ModelId; } - var toolOptions = imageGenerationOptions?.RawRepresentationFactory?.Invoke(null!) as Dictionary ?? new(); - toolOptions["type"] = "image_generation"; - // Size: Image dimensions (e.g., 1024x1024, 1024x1536) - if (imageGenerationOptions?.ImageSize is not null && !toolOptions.ContainsKey("size")) + if (imageGenerationOptions?.ImageSize is not null && result.Size is null) { // Use a custom type to ensure the size is formatted correctly. // This is a workaround for OpenAI's specific size format requirements. - toolOptions["size"] = new GeneratedImageSize( + result.Size = new ImageGenerationToolSize( imageGenerationOptions.ImageSize.Value.Width, - imageGenerationOptions.ImageSize.Value.Height).ToString(); + imageGenerationOptions.ImageSize.Value.Height); } // Format: File output format - if (imageGenerationOptions?.MediaType is not null && !toolOptions.ContainsKey("format")) + if (imageGenerationOptions?.MediaType is not null && result.OutputFileFormat is null) { - toolOptions["output_format"] = imageGenerationOptions.MediaType switch + result.OutputFileFormat = imageGenerationOptions.MediaType switch { - "image/png" => GeneratedImageFileFormat.Png.ToString(), - "image/jpeg" => GeneratedImageFileFormat.Jpeg.ToString(), - "image/webp" => GeneratedImageFileFormat.Webp.ToString(), - _ => string.Empty, + "image/png" => ImageGenerationToolOutputFileFormat.Png, + "image/jpeg" => ImageGenerationToolOutputFileFormat.Jpeg, + "image/webp" => ImageGenerationToolOutputFileFormat.Webp, + _ => null, }; } - // unexposed properties, string unless noted - // background: transparent, opaque, auto - // input_fidelity: effort model exerts to match input (high, low) - // input_image_mask: optional image mask for inpainting. Object with property file_id string or image_url data string. - // model: Model ID to use for image generation - // moderation: Moderation level (auto, low) - // output_compression: (int) Compression level (0-100%) for JPEG and WebP formats - // partial_images: (int) Number of partial images to return (0-3) - // quality: Rendering quality (e.g. low, medium, high) - - // Can't create the tool, but we can deserialize it from Json - BinaryData? toolOptionsData = BinaryData.FromBytes( - JsonSerializer.SerializeToUtf8Bytes(toolOptions, OpenAIJsonContext.Default.IDictionaryStringObject)); - return ModelReaderWriter.Read(toolOptionsData, ModelReaderWriterOptions.Json)!; + return result; } /// Creates a from a . @@ -620,7 +551,7 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt break; case HostedImageGenerationTool imageGenerationTool: - result.Tools.Add(ToImageResponseTool(imageGenerationTool, options)); + result.Tools.Add(ToImageResponseTool(imageGenerationTool)); break; case HostedWebSearchTool webSearchTool: From 6a6ffa24080af31d997957c88de0b9e261d77d51 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 3 Oct 2025 13:32:23 -0700 Subject: [PATCH 15/24] Temporary OpenAI feed --- NuGet.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NuGet.config b/NuGet.config index a8e7ecd2c9c..2b6a50c4605 100644 --- a/NuGet.config +++ b/NuGet.config @@ -18,7 +18,7 @@ - + @@ -41,7 +41,7 @@ - + From 94ceab2416d2e1bf210dc8515c89905a6c36550d Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 3 Oct 2025 13:32:35 -0700 Subject: [PATCH 16/24] Fix tests --- .../Utilities/AIJsonUtilities.Defaults.cs | 1 + .../ChatCompletion/ImageGeneratingChatClientTests.cs | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 4b8a4fb1576..8bf3a708c5b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -93,6 +93,7 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(IEnumerable))] [JsonSerializable(typeof(char))] [JsonSerializable(typeof(string))] + [JsonSerializable(typeof(string[]))] [JsonSerializable(typeof(int))] [JsonSerializable(typeof(short))] [JsonSerializable(typeof(long))] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs index b8e046d7263..209031b6461 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs @@ -92,16 +92,19 @@ public async Task GetResponseAsync_WithImageGenerationTool_ReplacesTool() Assert.True(innerClientCalled); Assert.NotNull(capturedOptions); Assert.NotNull(capturedOptions.Tools); - Assert.Equal(2, capturedOptions.Tools.Count); // Should have GenerateImage and EditImage functions + Assert.Equal(3, capturedOptions.Tools.Count); // Verify the functions are properly created var generateImageFunction = capturedOptions.Tools[0] as AIFunction; var editImageFunction = capturedOptions.Tools[1] as AIFunction; + var getImagesForEditImageFunction = capturedOptions.Tools[2] as AIFunction; Assert.NotNull(generateImageFunction); Assert.NotNull(editImageFunction); + Assert.NotNull(getImagesForEditImageFunction); Assert.Equal("GenerateImage", generateImageFunction.Name); Assert.Equal("EditImage", editImageFunction.Name); + Assert.Equal("GetImagesForEdit", getImagesForEditImageFunction.Name); } [Fact] @@ -137,7 +140,7 @@ public async Task GetResponseAsync_WithMixedTools_ReplacesOnlyImageGenerationToo Assert.True(innerClientCalled); Assert.NotNull(capturedOptions); Assert.NotNull(capturedOptions.Tools); - Assert.Equal(3, capturedOptions.Tools.Count); // DummyFunction + GenerateImage + EditImage + Assert.Equal(4, capturedOptions.Tools.Count); // DummyFunction + GenerateImage + EditImage + GetImagesForEdit Assert.Same(dummyFunction, capturedOptions.Tools[0]); // Original function preserved Assert.IsAssignableFrom(capturedOptions.Tools[1]); // GenerateImage function @@ -244,7 +247,7 @@ static async IAsyncEnumerable GetUpdatesAsync() Assert.True(innerClientCalled); Assert.NotNull(capturedOptions); Assert.NotNull(capturedOptions.Tools); - Assert.Equal(2, capturedOptions.Tools.Count); // GenerateImage and EditImage functions + Assert.Equal(3, capturedOptions.Tools.Count); } [Fact] From 96e9747c2220a98b9d7f4b89bb1ad99fdaaab483 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 3 Oct 2025 16:55:20 -0700 Subject: [PATCH 17/24] Add integration tests for ImageGeneratingChatClient --- ...ageGeneratingChatClientIntegrationTests.cs | 15 + ...ageGeneratingChatClientIntegrationTests.cs | 386 ++++++++++++++++++ ...ageGeneratingChatClientIntegrationTests.cs | 96 +++++ ...ageGeneratingChatClientIntegrationTests.cs | 15 + 4 files changed, 512 insertions(+) create mode 100644 test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..3a05f7fba9c --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI; + +/// +/// Azure AI Inference-specific integration tests for ImageGeneratingChatClient. +/// Tests the ImageGeneratingChatClient with Azure AI Inference chat client implementation. +/// +public class AzureAIInferenceImageGeneratingChatClientIntegrationTests : ImageGeneratingChatClientIntegrationTests +{ + protected override IChatClient? CreateChatClient() => + IntegrationTestHelpers.GetChatCompletionsClient() + ?.AsIChatClient(TestRunnerConfiguration.Instance["AzureAIInference:ChatModel"] ?? "gpt-4o-mini"); +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..28d44a5e1d3 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,386 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.TestUtilities; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope +#pragma warning disable CA2214 // Do not call overridable methods in constructors + +namespace Microsoft.Extensions.AI; + +/// +/// Abstract base class for integration tests that verify ImageGeneratingChatClient with real IChatClient implementations. +/// Concrete test classes should inherit from this and provide a real IChatClient that supports function calling. +/// +public abstract class ImageGeneratingChatClientIntegrationTests : IDisposable +{ + private readonly IChatClient? _baseChatClient; + + protected ImageGeneratingChatClientIntegrationTests() + { + _baseChatClient = CreateChatClient(); + ImageGenerator = CreateImageGenerator(); + + if (_baseChatClient != null) + { + ChatClient = _baseChatClient + .AsBuilder() + .UseImageGeneration(ImageGenerator) + .UseFunctionInvocation() + .Build(); + } + } + + /// Gets the ImageGeneratingChatClient configured with function invocation support. + protected IChatClient? ChatClient { get; } + + /// Gets the IImageGenerator used for testing. + protected IImageGenerator ImageGenerator { get; } + + public void Dispose() + { + ChatClient?.Dispose(); + _baseChatClient?.Dispose(); + ImageGenerator.Dispose(); + GC.SuppressFinalize(this); + } + + /// + /// Creates the base IChatClient implementation to test with. + /// Should return a real chat client that supports function calling. + /// + /// An IChatClient instance, or null to skip tests. + protected abstract IChatClient? CreateChatClient(); + + /// + /// Creates the IImageGenerator implementation for testing. + /// The default implementation creates a test image generator that captures calls. + /// + /// An IImageGenerator instance for testing. + protected virtual IImageGenerator CreateImageGenerator() => new CapturingImageGenerator(); + + [ConditionalFact] + public virtual async Task GenerateImage_CallsGenerateFunction_ReturnsDataContent() + { + SkipIfNotEnabled(); + + var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await ChatClient.GetResponseAsync( + [new ChatMessage(ChatRole.User, "Please generate an image of a cat")], + chatOptions); + + // Assert + Assert.Single(imageGenerator.GenerateCalls); + var (request, _) = imageGenerator.GenerateCalls[0]; + Assert.Contains("cat", request.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.Null(request.OriginalImages); // Generation, not editing + + // Verify that we get DataContent back in the response + var dataContents = response.Messages + .SelectMany(m => m.Contents) + .OfType() + .ToList(); + + Assert.Single(dataContents); + var imageContent = dataContents[0]; + Assert.Equal("image/png", imageContent.MediaType); + Assert.False(imageContent.Data.IsEmpty); + } + + [ConditionalFact] + public virtual async Task EditImage_WithImageInSameRequest_PassesExactDataContent() + { + SkipIfNotEnabled(); + + var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var testImageData = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG header + var originalImageData = new DataContent(testImageData, "image/png") { Name = "original.png" }; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await ChatClient.GetResponseAsync( + [new ChatMessage(ChatRole.User, [new TextContent("Please edit this image to add a red border"), originalImageData])], + chatOptions); + + // Assert + Assert.Single(imageGenerator.GenerateCalls); + var (request, _) = imageGenerator.GenerateCalls[0]; + Assert.NotNull(request.OriginalImages); + + var originalImage = Assert.Single(request.OriginalImages); + var originalImageContent = Assert.IsType(originalImage); + Assert.Equal(testImageData, originalImageContent.Data.ToArray()); + Assert.Equal("image/png", originalImageContent.MediaType); + Assert.Equal("original.png", originalImageContent.Name); + } + + [ConditionalFact] + public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage() + { + SkipIfNotEnabled(); + + var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + var chatHistory = new List + { + new(ChatRole.User, "Please generate an image of a dog") + }; + + // First request: Generate image + var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + chatHistory.AddRange(firstResponse.Messages); + + // Second request: Edit the generated image + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to make it more colorful")); + var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + + // Assert + Assert.Equal(2, imageGenerator.GenerateCalls.Count); + + // First call should be generation (no original images) + var (firstRequest, _) = imageGenerator.GenerateCalls[0]; + Assert.Contains("dog", firstRequest.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.Null(firstRequest.OriginalImages); + + // Second call should be editing (with original images) + var (secondRequest, _) = imageGenerator.GenerateCalls[1]; + Assert.Contains("colorful", secondRequest.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.NotNull(secondRequest.OriginalImages); + Assert.Single(secondRequest.OriginalImages); + + var editedImage = Assert.IsType(secondRequest.OriginalImages.First()); + Assert.Equal("image/png", editedImage.MediaType); + Assert.Contains("generated_image_1", editedImage.Name); + } + + [ConditionalFact] + public virtual async Task MultipleEdits_EditsLatestImage() + { + SkipIfNotEnabled(); + + var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + var chatHistory = new List + { + new(ChatRole.User, "Please generate an image of a tree") + }; + + // First: Generate image + var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + chatHistory.AddRange(firstResponse.Messages); + + // Second: First edit + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add flowers")); + var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + chatHistory.AddRange(secondResponse.Messages); + var firstEditedImage = secondResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .SingleOrDefault(); + + // Third: Second edit (should edit the latest version by default) + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add birds")); + var thirdResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + + // Assert + Assert.Equal(3, imageGenerator.GenerateCalls.Count); + + // Third call should edit the second generated image (from first edit), not the original + var (thirdRequest, _) = imageGenerator.GenerateCalls[2]; + Assert.Contains("birds", thirdRequest.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.NotNull(thirdRequest.OriginalImages); + + var lastImageToEdit = Assert.IsType(thirdRequest.OriginalImages.First()); + Assert.Equal(firstEditedImage, lastImageToEdit); + } + + [ConditionalFact] + public virtual async Task MultipleEdits_EditsFirstImage() + { + SkipIfNotEnabled(); + + var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + var chatHistory = new List + { + new(ChatRole.User, "Please generate an image of a tree") + }; + + // First: Generate image + var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + chatHistory.AddRange(firstResponse.Messages); + var firstGeneratedImage = firstResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .SingleOrDefault(); + + // Second: First edit + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add fruit")); + var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + chatHistory.AddRange(secondResponse.Messages); + + // Third: Second edit (should edit the latest version by default) + chatHistory.Add(new ChatMessage(ChatRole.User, "That didn't work out. Please edit the original image to add birds")); + var thirdResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + + // Assert + Assert.Equal(3, imageGenerator.GenerateCalls.Count); + + // Third call should edit the original generated image (not from edit) + var (thirdRequest, _) = imageGenerator.GenerateCalls[2]; + Assert.Contains("birds", thirdRequest.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.NotNull(thirdRequest.OriginalImages); + + var lastImageToEdit = Assert.IsType(thirdRequest.OriginalImages.First()); + Assert.Equal(firstGeneratedImage, lastImageToEdit); + } + + [ConditionalFact] + public virtual async Task ImageGeneration_WithOptions_PassesOptionsToGenerator() + { + SkipIfNotEnabled(); + + var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerationOptions = new ImageGenerationOptions + { + Count = 2, + ImageSize = new System.Drawing.Size(512, 512) + }; + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool { Options = imageGenerationOptions }] + }; + + // Act + var response = await ChatClient.GetResponseAsync( + [new ChatMessage(ChatRole.User, "Generate an image of a castle")], + chatOptions); + + // Assert + Assert.Single(imageGenerator.GenerateCalls); + var (_, options) = imageGenerator.GenerateCalls[0]; + Assert.NotNull(options); + Assert.Equal(2, options.Count); + Assert.Equal(new System.Drawing.Size(512, 512), options.ImageSize); + } + + [ConditionalFact] + public virtual async Task ImageContentHandling_AllImages_ReplacesImagesWithPlaceholders() + { + SkipIfNotEnabled(); + + var testImageData = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG header + var capturedMessages = new List>(); + + // Create a new ImageGeneratingChatClient with AllImages data content handling + + using var imageGeneratingClient = _baseChatClient! + .AsBuilder() + .UseImageGeneration(ImageGenerator) + .Use((messages, options, next, cancellationToken) => + { + capturedMessages.Add(messages); + return next(messages, options, cancellationToken); + }) + .UseFunctionInvocation() + .Build(); + + var originalImage = new DataContent(testImageData, "image/png") { Name = "test.png" }; + + // Act + await imageGeneratingClient.GetResponseAsync([ + new ChatMessage(ChatRole.User, [ + new TextContent("Here's an image to process"), + originalImage + ]) + ], new ChatOptions { Tools = [new HostedImageGenerationTool()] }); + + // Assert + Assert.NotEmpty(capturedMessages); + var processedMessages = capturedMessages.First().ToList(); + var userMessage = processedMessages.First(m => m.Role == ChatRole.User); + + // Should have text content with placeholder instead of original image + var textContents = userMessage.Contents.OfType().ToList(); + Assert.Contains(textContents, tc => tc.Text.Contains("[meai_image:") && tc.Text.Contains("] available for edit")); + + // Should not contain the original DataContent + Assert.DoesNotContain(userMessage.Contents, c => c == originalImage); + } + + /// + /// Test image generator that captures calls and returns fake image data. + /// + protected sealed class CapturingImageGenerator : IImageGenerator + { + private const string TestImageMediaType = "image/png"; + private static readonly byte[] _testImageData = [0x89, 0x50, 0x4E, 0x47]; // PNG header + + public List<(ImageGenerationRequest request, ImageGenerationOptions? options)> GenerateCalls { get; } = []; + public int ImageCounter { get; private set; } + + public Task GenerateAsync(ImageGenerationRequest request, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + GenerateCalls.Add((request, options)); + + // Create fake image data with unique content + var imageData = new byte[_testImageData.Length + 4]; + _testImageData.CopyTo(imageData, 0); + BitConverter.GetBytes(++ImageCounter).CopyTo(imageData, _testImageData.Length); + + var imageContent = new DataContent(imageData, TestImageMediaType) + { + Name = $"generated_image_{ImageCounter}.png" + }; + + return Task.FromResult(new ImageGenerationResponse([imageContent])); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + // No resources to dispose + } + } + + [MemberNotNull(nameof(ChatClient))] + protected void SkipIfNotEnabled() + { + string? skipIntegration = TestRunnerConfiguration.Instance["SkipIntegrationTests"]; + + if (skipIntegration is not null || ChatClient is null) + { + throw new SkipTestException("Client is not enabled."); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..b58c8b0c0ea --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,96 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading.Tasks; +using Microsoft.TestUtilities; +using OllamaSharp; + +namespace Microsoft.Extensions.AI; + +/// +/// OllamaSharp-specific integration tests for ImageGeneratingChatClient. +/// Tests the ImageGeneratingChatClient with OllamaSharp chat client implementation. +/// +public class OllamaSharpImageGeneratingChatClientIntegrationTests : ImageGeneratingChatClientIntegrationTests +{ + protected override IChatClient? CreateChatClient() => + IntegrationTestHelpers.GetOllamaUri() is Uri endpoint ? + new OllamaApiClient(endpoint, "llama3.2") : + null; + + // Note: Some Ollama models may have limitations with function calling. + // These tests may need to be skipped or use different models if function calling is not supported. + public override Task GenerateImage_CallsGenerateFunction_ReturnsDataContent() + { + // Skip if the current Ollama model doesn't support function calling well + try + { + return base.GenerateImage_CallsGenerateFunction_ReturnsDataContent(); + } + catch + { + throw new SkipTestException("Ollama model may not support the required function calling for image generation."); + } + } + + public override Task EditImage_WithImageInSameRequest_PassesExactDataContent() + { + try + { + return base.EditImage_WithImageInSameRequest_PassesExactDataContent(); + } + catch + { + throw new SkipTestException("Ollama model may not support the required function calling for image editing."); + } + } + + public override Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage() + { + try + { + return base.GenerateThenEdit_FromChatHistory_EditsGeneratedImage(); + } + catch + { + throw new SkipTestException("Ollama model may not support complex function calling workflows."); + } + } + + public override Task MultipleEdits_EditsLatestImage() + { + try + { + return base.MultipleEdits_EditsLatestImage(); + } + catch + { + throw new SkipTestException("Ollama model may not support complex function calling workflows."); + } + } + + public override Task ImageGeneration_WithOptions_PassesOptionsToGenerator() + { + try + { + return base.ImageGeneration_WithOptions_PassesOptionsToGenerator(); + } + catch + { + throw new SkipTestException("Ollama model may not support function calling with complex options."); + } + } + + public override Task ImageContentHandling_AllImages_ReplacesImagesWithPlaceholders() + { + try + { + return base.ImageContentHandling_AllImages_ReplacesImagesWithPlaceholders(); + } + catch + { + throw new SkipTestException("Ollama model may not support complex data content handling workflows."); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..7f9e6195aa3 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI; + +/// +/// OpenAI-specific integration tests for ImageGeneratingChatClient. +/// Tests the ImageGeneratingChatClient with OpenAI's chat client implementation. +/// +public class OpenAIImageGeneratingChatClientIntegrationTests : ImageGeneratingChatClientIntegrationTests +{ + protected override IChatClient? CreateChatClient() => + IntegrationTestHelpers.GetOpenAIClient() + ?.GetChatClient(TestRunnerConfiguration.Instance["OpenAI:ChatModel"] ?? "gpt-4o-mini").AsIChatClient(); +} From 9ddc91ab2e40c5d602a6bd986088939bd006cb0f Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Fri, 3 Oct 2025 17:56:47 -0700 Subject: [PATCH 18/24] Remove ChatRole.Tool -> Assistant workaround --- .../ImageGeneratingChatClient.cs | 34 ++++++++++++------ ...ageGeneratingChatClientIntegrationTests.cs | 36 +++++++------------ 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index e9c372c6ab6..c766042f68b 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -116,8 +116,7 @@ public override async Task GetResponseAsync( // Replace FunctionResultContent instances with generated image content foreach (var message in response.Messages) { - var newContents = ReplaceImageGenerationFunctionResults(message.Contents); - message.Contents = newContents; + message.Contents = ReplaceImageGenerationFunctionResults(message.Contents); } return response; @@ -236,11 +235,6 @@ private IEnumerable ProcessChatMessages(IEnumerable me newMessages ??= new List(messages.Take(messageIndex)); var newMessage = message.Clone(); - if (newMessage.Role == ChatRole.Tool) - { - // workaround: the chat client will ignore tool messages, so change the role to assistant - newMessage.Role = ChatRole.Assistant; - } newMessage.Contents = newContents; newMessages.Add(newMessage); @@ -330,7 +324,15 @@ private string StoreImage(DataContent imageContent, bool isGenerated = false) return imageId; } - /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. + /// + /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. + /// We will have two messages + /// 1. Role: Assistant, FunctionCall + /// 2. Role: Tool, FunctionResult + /// We need to content from both but we shouldn't remove the messages. + /// If we do not then ChatClient's may not accept our altered history. + /// A better approach might be to recreate history that matches what a server side HostedImageGenerationTool would have produced. + /// /// The list of AI content to process. private IList ReplaceImageGenerationFunctionResults(IList contents) { @@ -344,15 +346,18 @@ private IList ReplaceImageGenerationFunctionResults(IList if (content is FunctionCallContent functionCall && _functionNames.Contains(functionCall.Name)) { - // create a new list and skip this + // create a new list and omit the FunctionCallContent newContents ??= CopyList(contents, i); + + // add a placeholder text content to avoid empty contents which could cause the client to drop the message + newContents.Add(new TextContent(string.Empty)); } else if (content is FunctionResultContent functionResult && _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) { newContents ??= CopyList(contents, i, imageContents.Count - 1); - // Insert generated image content in its place + // Insert generated image content in its place, do not preserve the FunctionResultContent foreach (var imageContent in imageContents) { newContents.Add(imageContent); @@ -411,6 +416,15 @@ private async Task GenerateImageAsync( [Description("Lists the identifiers of all images available for edit.")] private string[] GetImagesForEdit() { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return ["No call ID available for image editing."]; + } + + _imageContentByCallId[callId] = []; + return _imageContentById.Keys.ToArray(); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs index 28d44a5e1d3..16724ba6003 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs @@ -91,11 +91,9 @@ [new ChatMessage(ChatRole.User, "Please generate an image of a cat")], // Verify that we get DataContent back in the response var dataContents = response.Messages .SelectMany(m => m.Contents) - .OfType() - .ToList(); + .OfType(); - Assert.Single(dataContents); - var imageContent = dataContents[0]; + var imageContent = Assert.Single(dataContents); Assert.Equal("image/png", imageContent.MediaType); Assert.False(imageContent.Data.IsEmpty); } @@ -119,8 +117,7 @@ [new ChatMessage(ChatRole.User, [new TextContent("Please edit this image to add chatOptions); // Assert - Assert.Single(imageGenerator.GenerateCalls); - var (request, _) = imageGenerator.GenerateCalls[0]; + var (request, _) = Assert.Single(imageGenerator.GenerateCalls); Assert.NotNull(request.OriginalImages); var originalImage = Assert.Single(request.OriginalImages); @@ -159,14 +156,15 @@ public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage() // First call should be generation (no original images) var (firstRequest, _) = imageGenerator.GenerateCalls[0]; - Assert.Contains("dog", firstRequest.Prompt, StringComparison.OrdinalIgnoreCase); Assert.Null(firstRequest.OriginalImages); + var firstContent = Assert.Single(firstResponse.Messages.SelectMany(m => m.Contents).OfType()); // Second call should be editing (with original images) var (secondRequest, _) = imageGenerator.GenerateCalls[1]; - Assert.Contains("colorful", secondRequest.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.Single(secondResponse.Messages.SelectMany(m => m.Contents).OfType()); Assert.NotNull(secondRequest.OriginalImages); - Assert.Single(secondRequest.OriginalImages); + var editContent = Assert.Single(secondRequest.OriginalImages); + Assert.Equal(firstContent, editContent); // Should be the same image as generated in first call var editedImage = Assert.IsType(secondRequest.OriginalImages.First()); Assert.Equal("image/png", editedImage.MediaType); @@ -197,13 +195,9 @@ public virtual async Task MultipleEdits_EditsLatestImage() chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add flowers")); var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); chatHistory.AddRange(secondResponse.Messages); - var firstEditedImage = secondResponse.Messages - .SelectMany(m => m.Contents) - .OfType() - .SingleOrDefault(); // Third: Second edit (should edit the latest version by default) - chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add birds")); + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit that image to add birds")); var thirdResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); // Assert @@ -211,11 +205,10 @@ public virtual async Task MultipleEdits_EditsLatestImage() // Third call should edit the second generated image (from first edit), not the original var (thirdRequest, _) = imageGenerator.GenerateCalls[2]; - Assert.Contains("birds", thirdRequest.Prompt, StringComparison.OrdinalIgnoreCase); Assert.NotNull(thirdRequest.OriginalImages); - - var lastImageToEdit = Assert.IsType(thirdRequest.OriginalImages.First()); - Assert.Equal(firstEditedImage, lastImageToEdit); + var secondImage = Assert.Single(secondResponse.Messages.SelectMany(m => m.Contents).OfType()); + var lastImageToEdit = Assert.Single(thirdRequest.OriginalImages.OfType()); + Assert.Equal(secondImage, lastImageToEdit); } [ConditionalFact] @@ -237,10 +230,6 @@ public virtual async Task MultipleEdits_EditsFirstImage() // First: Generate image var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); chatHistory.AddRange(firstResponse.Messages); - var firstGeneratedImage = firstResponse.Messages - .SelectMany(m => m.Contents) - .OfType() - .SingleOrDefault(); // Second: First edit chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add fruit")); @@ -256,9 +245,8 @@ public virtual async Task MultipleEdits_EditsFirstImage() // Third call should edit the original generated image (not from edit) var (thirdRequest, _) = imageGenerator.GenerateCalls[2]; - Assert.Contains("birds", thirdRequest.Prompt, StringComparison.OrdinalIgnoreCase); Assert.NotNull(thirdRequest.OriginalImages); - + var firstGeneratedImage = Assert.Single(firstResponse.Messages.SelectMany(m => m.Contents).OfType()); var lastImageToEdit = Assert.IsType(thirdRequest.OriginalImages.First()); Assert.Equal(firstGeneratedImage, lastImageToEdit); } From 3b589acf61742ed3e57a862fe74291ce2b444fb2 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 6 Oct 2025 14:48:05 -0700 Subject: [PATCH 19/24] Remove use of private reflection for Image results --- .../OpenAIResponsesChatClient.cs | 69 ++++--------------- 1 file changed, 14 insertions(+), 55 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 798fe1c3a90..263d556b637 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -10,7 +10,6 @@ using System.Reflection; using System.Runtime.CompilerServices; using System.Text.Json; -using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; @@ -124,7 +123,7 @@ internal static ChatResponse FromOpenAIResponse(OpenAIResponse openAIResponse, R if (openAIResponse.OutputItems is not null) { - response.Messages = [.. ToChatMessages(openAIResponse.OutputItems)]; + response.Messages = [.. ToChatMessages(openAIResponse.OutputItems, openAIOptions)]; if (response.Messages.LastOrDefault() is { } lastMessage && openAIResponse.Error is { } error) { @@ -140,7 +139,7 @@ internal static ChatResponse FromOpenAIResponse(OpenAIResponse openAIResponse, R return response; } - internal static IEnumerable ToChatMessages(IEnumerable items) + internal static IEnumerable ToChatMessages(IEnumerable items, ResponseCreationOptions? options = null) { ChatMessage? message = null; @@ -201,7 +200,7 @@ internal static IEnumerable ToChatMessages(IEnumerable ToChatMessages(IEnumerable? additionalRawData = typeof(ResponseItem) - .GetProperty("SerializedAdditionalRawData", InternalBindingFlags) - ?.GetValue(outputItem) as IDictionary; - - string outputFormat = getStringProperty("output_format") ?? "png"; + var imageGenTool = options?.Tools.OfType().FirstOrDefault(); + string outputFormat = imageGenTool?.OutputFileFormat?.ToString() ?? "png"; return new DataContent(outputItem.GeneratedImageBytes, $"image/{outputFormat}") { RawRepresentation = outputItem }; - - string? getStringProperty(string name) - { - if (additionalRawData?.TryGetValue(name, out var outputFormat) == true) - { - var stringJsonTypeInfo = (JsonTypeInfo)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string)); - return JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo); - } - - return null; - } } - [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(ResponseItem))] - private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResponseImageGenerationCallPartialImageUpdate update) + private static DataContent GetContentFromImageGenPartialImageEvent(StreamingResponseImageGenerationCallPartialImageUpdate update, ResponseCreationOptions? options) { - const BindingFlags InternalBindingFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; - - IDictionary? additionalRawData = typeof(ResponseItem) - .GetProperty("SerializedAdditionalRawData", InternalBindingFlags) - ?.GetValue(update) as IDictionary; + var imageGenTool = options?.Tools.OfType().FirstOrDefault(); + var outputType = imageGenTool?.OutputFileFormat?.ToString() ?? "png"; - // Properties - // background - // output_format - // quality - // revised_prompt - // size - - string outputFormat = getStringProperty("output_format") ?? "png"; - - return new DataContent(update.PartialImageBytes, $"image/{outputFormat}") + return new DataContent(update.PartialImageBytes, $"image/{outputType}") { RawRepresentation = update, AdditionalProperties = new() { - ["ItemId"] = update.ItemId, - ["OutputIndex"] = update.OutputIndex, - ["PartialImageIndex"] = update.PartialImageIndex + [nameof(update.ItemId)] = update.ItemId, + [nameof(update.OutputIndex)] = update.OutputIndex, + [nameof(update.PartialImageIndex)] = update.PartialImageIndex } }; - - string? getStringProperty(string name) - { - if (additionalRawData?.TryGetValue(name, out var outputFormat) == true) - { - var stringJsonTypeInfo = (JsonTypeInfo)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string)); - return JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo); - } - - return null; - } } /// @@ -426,7 +385,7 @@ outputItemDoneUpdate.Item is MessageResponseItem mri && break; case StreamingResponseImageGenerationCallPartialImageUpdate streamingImageGenUpdate: - yield return CreateUpdate(GetContentFromImageGenPartialImageEvent(streamingImageGenUpdate)); + yield return CreateUpdate(GetContentFromImageGenPartialImageEvent(streamingImageGenUpdate, options)); break; default: From 20919abbdb121c08107a0e69e4402468f1b2c7d0 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 6 Oct 2025 16:27:29 -0700 Subject: [PATCH 20/24] Add ChatResponseUpdate.Clone --- .../ChatCompletion/ChatResponseUpdate.cs | 24 +++ .../ImageGeneratingChatClient.cs | 20 +-- .../ChatCompletion/ChatResponseUpdateTests.cs | 161 ++++++++++++++++++ 3 files changed, 192 insertions(+), 13 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs index f54d1cd0ea9..b5e87a15af3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs @@ -58,6 +58,30 @@ public ChatResponseUpdate(ChatRole? role, IList? contents) _contents = contents; } + /// + /// Creates a new ChatResponseUpdate instance that is a copy of the current object. + /// + /// The cloned object is a shallow copy; reference-type properties will reference the same + /// objects as the original. Use this method to duplicate the response update for further modification without + /// affecting the original instance. + /// A new ChatResponseUpdate object with the same property values as the current instance. + [Experimental("MEAI001")] + public ChatResponseUpdate Clone() => + new() + { + AdditionalProperties = AdditionalProperties, + AuthorName = AuthorName, + Contents = Contents, + CreatedAt = CreatedAt, + ConversationId = ConversationId, + FinishReason = FinishReason, + MessageId = MessageId, + ModelId = ModelId, + RawRepresentation = RawRepresentation, + ResponseId = ResponseId, + Role = Role, + }; + /// Gets or sets the name of the author of the response update. public string? AuthorName { diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index c766042f68b..37a0aa5c93f 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -150,19 +150,11 @@ public override async IAsyncEnumerable GetStreamingResponseA // Replace any FunctionResultContent instances with generated image content var newContents = ReplaceImageGenerationFunctionResults(update.Contents); - if (newContents != update.Contents) + if (!ReferenceEquals(newContents, update.Contents)) { // Create a new update instance with modified contents - var modifiedUpdate = new ChatResponseUpdate(update.Role, newContents) - { - AuthorName = update.AuthorName, - RawRepresentation = update.RawRepresentation, - AdditionalProperties = update.AdditionalProperties, - ResponseId = update.ResponseId, - MessageId = update.MessageId, - ConversationId = update.ConversationId - }; - + var modifiedUpdate = update.Clone(); + modifiedUpdate.Contents = newContents; yield return modifiedUpdate; } else @@ -329,9 +321,11 @@ private string StoreImage(DataContent imageContent, bool isGenerated = false) /// We will have two messages /// 1. Role: Assistant, FunctionCall /// 2. Role: Tool, FunctionResult - /// We need to content from both but we shouldn't remove the messages. + /// We need to replace content from both but we shouldn't remove the messages. /// If we do not then ChatClient's may not accept our altered history. - /// A better approach might be to recreate history that matches what a server side HostedImageGenerationTool would have produced. + /// + /// When interating with a HostedImageGenerationTool we will have typically only see a single Message with + /// Role: Assistant that contains the DataContent (or a provider specific content, that's exposed as DataContent). /// /// The list of AI content to process. private IList ReplaceImageGenerationFunctionResults(IList contents) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs index 413215d9a44..9727a58ac47 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs @@ -167,4 +167,165 @@ public void JsonSerialization_Roundtrips() Assert.IsType(value); Assert.Equal("value", ((JsonElement)value!).GetString()); } + + [Fact] + public void Clone_CreatesShallowCopy() + { + // Arrange + var originalAdditionalProperties = new AdditionalPropertiesDictionary { ["key"] = "value" }; + var originalContents = new List { new TextContent("text1"), new TextContent("text2") }; + var originalRawRepresentation = new object(); + var originalCreatedAt = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero); + + var original = new ChatResponseUpdate + { + AdditionalProperties = originalAdditionalProperties, + AuthorName = "author", + Contents = originalContents, + CreatedAt = originalCreatedAt, + ConversationId = "conv123", + FinishReason = ChatFinishReason.ContentFilter, + MessageId = "msg456", + ModelId = "model789", + RawRepresentation = originalRawRepresentation, + ResponseId = "resp012", + Role = ChatRole.Assistant, + }; + + // Act + var clone = original.Clone(); + + // Assert - Different instances + Assert.NotSame(original, clone); + + // Assert - All properties copied correctly + Assert.Equal(original.AuthorName, clone.AuthorName); + Assert.Equal(original.Role, clone.Role); + Assert.Equal(original.CreatedAt, clone.CreatedAt); + Assert.Equal(original.ConversationId, clone.ConversationId); + Assert.Equal(original.FinishReason, clone.FinishReason); + Assert.Equal(original.MessageId, clone.MessageId); + Assert.Equal(original.ModelId, clone.ModelId); + Assert.Equal(original.ResponseId, clone.ResponseId); + + // Assert - Reference properties are shallow copied (same references) + Assert.Same(original.AdditionalProperties, clone.AdditionalProperties); + Assert.Same(original.Contents, clone.Contents); + Assert.Same(original.RawRepresentation, clone.RawRepresentation); + } + + [Fact] + public void Clone_WithNullProperties_CopiesCorrectly() + { + // Arrange + var original = new ChatResponseUpdate + { + Role = ChatRole.User, + ResponseId = "resp123" + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotSame(original, clone); + Assert.Equal(ChatRole.User, clone.Role); + Assert.Equal("resp123", clone.ResponseId); + Assert.Null(clone.AdditionalProperties); + Assert.Null(clone.AuthorName); + Assert.Null(clone.CreatedAt); + Assert.Null(clone.ConversationId); + Assert.Null(clone.FinishReason); + Assert.Null(clone.MessageId); + Assert.Null(clone.ModelId); + Assert.Null(clone.RawRepresentation); + Assert.Empty(clone.Contents); // Contents property initializes to empty list + } + + [Fact] + public void Clone_WithDefaultConstructor_CopiesCorrectly() + { + // Arrange + var original = new ChatResponseUpdate(); + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotSame(original, clone); + Assert.Null(clone.AuthorName); + Assert.Null(clone.Role); + Assert.Empty(clone.Contents); + Assert.Null(clone.RawRepresentation); + Assert.Null(clone.AdditionalProperties); + Assert.Null(clone.ResponseId); + Assert.Null(clone.MessageId); + Assert.Null(clone.CreatedAt); + Assert.Null(clone.FinishReason); + Assert.Null(clone.ConversationId); + Assert.Null(clone.ModelId); + } + + [Fact] + public void Clone_ModifyingClone_DoesNotAffectOriginal() + { + // Arrange + var original = new ChatResponseUpdate + { + AuthorName = "original_author", + Role = ChatRole.User, + ResponseId = "original_id", + ModelId = "original_model" + }; + + // Act + var clone = original.Clone(); + clone.AuthorName = "modified_author"; + clone.Role = ChatRole.Assistant; + clone.ResponseId = "modified_id"; + clone.ModelId = "modified_model"; + + // Assert - Original remains unchanged + Assert.Equal("original_author", original.AuthorName); + Assert.Equal(ChatRole.User, original.Role); + Assert.Equal("original_id", original.ResponseId); + Assert.Equal("original_model", original.ModelId); + + // Assert - Clone has modified values + Assert.Equal("modified_author", clone.AuthorName); + Assert.Equal(ChatRole.Assistant, clone.Role); + Assert.Equal("modified_id", clone.ResponseId); + Assert.Equal("modified_model", clone.ModelId); + } + + [Fact] + public void Clone_ModifyingSharedReferences_AffectsBothInstances() + { + // Arrange + var sharedAdditionalProperties = new AdditionalPropertiesDictionary { ["initial"] = "value" }; + var sharedContents = new List { new TextContent("initial") }; + + var original = new ChatResponseUpdate + { + AdditionalProperties = sharedAdditionalProperties, + Contents = sharedContents + }; + + // Act + var clone = original.Clone(); + + // Modify the shared reference objects + sharedAdditionalProperties["modified"] = "new_value"; + sharedContents.Add(new TextContent("added")); + + // Assert - Both original and clone are affected due to shallow copy + Assert.Same(original.AdditionalProperties, clone.AdditionalProperties); + Assert.Same(original.Contents, clone.Contents); + Assert.Equal(2, original.AdditionalProperties.Count); + Assert.Equal(2, clone.AdditionalProperties?.Count); + Assert.Equal(2, original.Contents.Count); + Assert.Equal(2, clone.Contents.Count); + Assert.True(original.AdditionalProperties.ContainsKey("modified")); + Assert.True(clone.AdditionalProperties?.ContainsKey("modified")); + } } From e5f68a6b3c5630ac3771c1c92cfcdf90e417c149 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 7 Oct 2025 10:34:35 -0700 Subject: [PATCH 21/24] Move all mutable state into RequestState object --- .../ImageGeneratingChatClient.cs | 571 +++++++++--------- 1 file changed, 282 insertions(+), 289 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index 37a0aa5c93f..c7a8c3cea02 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -51,28 +51,8 @@ public enum DataContentHandling private const string ImageKey = "meai_image"; private readonly IImageGenerator _imageGenerator; - private readonly AITool[] _aiTools; - private readonly HashSet _functionNames; private readonly DataContentHandling _dataContentHandling; - // the following fields all have scope per-request. They are cleared at the start of each request. - private readonly Dictionary> _imageContentByCallId = []; - private readonly Dictionary _imageContentById = new(StringComparer.OrdinalIgnoreCase); - private ImageGenerationOptions? _imageGenerationOptions; - - private static List CopyList(IList original, int toOffsetExclusive, int additionalCapacity = 0) - { - var newList = new List(original.Count + additionalCapacity); - - // Copy all items up to and excluding the current index - for (int j = 0; j < toOffsetExclusive; j++) - { - newList.Add(original[j]); - } - - return newList; - } - /// Initializes a new instance of the class. /// The underlying . /// An instance that will be used for image generation operations. @@ -83,14 +63,6 @@ public ImageGeneratingChatClient(IChatClient innerClient, IImageGenerator imageG : base(innerClient) { _imageGenerator = Throw.IfNull(imageGenerator); - _aiTools = - [ - AIFunctionFactory.Create(GenerateImageAsync), - AIFunctionFactory.Create(EditImageAsync), - AIFunctionFactory.Create(GetImagesForEdit) - ]; - - _functionNames = new(_aiTools.Select(t => t.Name), StringComparer.Ordinal); _dataContentHandling = dataContentHandling; } @@ -100,33 +72,22 @@ public override async Task GetResponseAsync( { _ = Throw.IfNull(messages); - // Clear any existing generated content for this request - _imageContentByCallId.Clear(); - _imageContentById.Clear(); + var requestState = new RequestState(_imageGenerator, _dataContentHandling); - try - { - // Process the chat options to replace HostedImageGenerationTool with functions - var processedOptions = ProcessChatOptions(options); - var processedMessages = ProcessChatMessages(messages); + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = requestState.ProcessChatOptions(options); + var processedMessages = requestState.ProcessChatMessages(messages); - // Get response from base implementation - var response = await base.GetResponseAsync(processedMessages, processedOptions, cancellationToken); - - // Replace FunctionResultContent instances with generated image content - foreach (var message in response.Messages) - { - message.Contents = ReplaceImageGenerationFunctionResults(message.Contents); - } + // Get response from base implementation + var response = await base.GetResponseAsync(processedMessages, processedOptions, cancellationToken); - return response; - } - finally + // Replace FunctionResultContent instances with generated image content + foreach (var message in response.Messages) { - // Clear any existing generated content for this request - _imageContentByCallId.Clear(); - _imageContentById.Clear(); + message.Contents = requestState.ReplaceImageGenerationFunctionResults(message.Contents); } + + return response; } /// @@ -135,39 +96,28 @@ public override async IAsyncEnumerable GetStreamingResponseA { _ = Throw.IfNull(messages); - // Clear any existing generated content for this request - _imageContentByCallId.Clear(); - _imageContentById.Clear(); + var requestState = new RequestState(_imageGenerator, _dataContentHandling); + + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = requestState.ProcessChatOptions(options); + var processedMessages = requestState.ProcessChatMessages(messages); - try + await foreach (var update in base.GetStreamingResponseAsync(processedMessages, processedOptions, cancellationToken)) { - // Process the chat options to replace HostedImageGenerationTool with functions - var processedOptions = ProcessChatOptions(options); - var processedMessages = ProcessChatMessages(messages); + // Replace any FunctionResultContent instances with generated image content + var newContents = requestState.ReplaceImageGenerationFunctionResults(update.Contents); - await foreach (var update in base.GetStreamingResponseAsync(processedMessages, processedOptions, cancellationToken)) + if (!ReferenceEquals(newContents, update.Contents)) { - // Replace any FunctionResultContent instances with generated image content - var newContents = ReplaceImageGenerationFunctionResults(update.Contents); - - if (!ReferenceEquals(newContents, update.Contents)) - { - // Create a new update instance with modified contents - var modifiedUpdate = update.Clone(); - modifiedUpdate.Contents = newContents; - yield return modifiedUpdate; - } - else - { - yield return update; - } + // Create a new update instance with modified contents + var modifiedUpdate = update.Clone(); + modifiedUpdate.Contents = newContents; + yield return modifiedUpdate; + } + else + { + yield return update; } - } - finally - { - // Clear any existing generated content for this request - _imageContentByCallId.Clear(); - _imageContentById.Clear(); } } @@ -183,277 +133,256 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } - private IEnumerable ProcessChatMessages(IEnumerable messages) + /// + /// Contains all the per-request state and methods for handling image generation requests. + /// This class is created fresh for each request to ensure thread safety. + /// This class is not exposed publicly and does not own any of it's resources. + /// + private sealed class RequestState { - // If no special handling is needed, return the original messages - if (_dataContentHandling == DataContentHandling.None) + private readonly IImageGenerator _imageGenerator; + private readonly DataContentHandling _dataContentHandling; + private readonly Dictionary> _imageContentByCallId = []; + private readonly Dictionary _imageContentById = new(StringComparer.OrdinalIgnoreCase); + private ImageGenerationOptions? _imageGenerationOptions; + + public RequestState(IImageGenerator imageGenerator, DataContentHandling dataContentHandling) { - return messages; + _imageGenerator = imageGenerator; + _dataContentHandling = dataContentHandling; } - List? newMessages = null; - int messageIndex = 0; - foreach (var message in messages) + private static List CopyList(IList original, int toOffsetExclusive, int additionalCapacity = 0) { - List? newContents = null; - for (int contentIndex = 0; contentIndex < message.Contents.Count; contentIndex++) + var newList = new List(original.Count + additionalCapacity); + + // Copy all items up to and excluding the current index + for (int j = 0; j < toOffsetExclusive; j++) { - var content = message.Contents[contentIndex]; - if (content is DataContent dataContent && dataContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + newList.Add(original[j]); + } + + return newList; + } + + private AITool[] GetAITools() + { + return + [ + AIFunctionFactory.Create(GenerateImageAsync), + AIFunctionFactory.Create(EditImageAsync), + AIFunctionFactory.Create(GetImagesForEdit) + ]; + } + + public IEnumerable ProcessChatMessages(IEnumerable messages) + { + // If no special handling is needed, return the original messages + if (_dataContentHandling == DataContentHandling.None) + { + return messages; + } + + List? newMessages = null; + int messageIndex = 0; + foreach (var message in messages) + { + List? newContents = null; + for (int contentIndex = 0; contentIndex < message.Contents.Count; contentIndex++) { - bool isGeneratedImage = dataContent.AdditionalProperties?.ContainsKey(ImageKey) == true; - if (_dataContentHandling == DataContentHandling.AllImages || - (_dataContentHandling == DataContentHandling.GeneratedImages && isGeneratedImage)) + var content = message.Contents[contentIndex]; + if (content is DataContent dataContent && dataContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) { - // Replace image with a placeholder text content - var imageId = StoreImage(dataContent); - - newContents ??= CopyList(message.Contents, contentIndex); - newContents.Add(new TextContent($"[{ImageKey}:{imageId}] available for edit.") + bool isGeneratedImage = dataContent.AdditionalProperties?.ContainsKey(ImageKey) == true; + if (_dataContentHandling == DataContentHandling.AllImages || + (_dataContentHandling == DataContentHandling.GeneratedImages && isGeneratedImage)) { - Annotations = dataContent.Annotations, - AdditionalProperties = dataContent.AdditionalProperties - }); - continue; // Skip adding the original content + // Replace image with a placeholder text content + var imageId = StoreImage(dataContent); + + newContents ??= CopyList(message.Contents, contentIndex); + newContents.Add(new TextContent($"[{ImageKey}:{imageId}] available for edit.") + { + Annotations = dataContent.Annotations, + AdditionalProperties = dataContent.AdditionalProperties + }); + continue; // Skip adding the original content + } } + + // Add the original content if no replacement was made + newContents?.Add(content); } - // Add the original content if no replacement was made - newContents?.Add(content); - } + if (newContents != null) + { + newMessages ??= new List(messages.Take(messageIndex)); - if (newContents != null) - { - newMessages ??= new List(messages.Take(messageIndex)); + var newMessage = message.Clone(); - var newMessage = message.Clone(); + newMessage.Contents = newContents; + newMessages.Add(newMessage); + } + else + { + newMessages?.Add(message); - newMessage.Contents = newContents; - newMessages.Add(newMessage); - } - else - { - newMessages?.Add(message); + } + messageIndex++; } - messageIndex++; - } - - return newMessages ?? messages; - } - - private ChatOptions? ProcessChatOptions(ChatOptions? options) - { - if (options?.Tools is null || options.Tools.Count == 0) - { - return options; + return newMessages ?? messages; } - List? newTools = null; - var tools = options.Tools; - for (int i = 0; i < tools.Count; i++) + public ChatOptions? ProcessChatOptions(ChatOptions? options) { - var tool = tools[i]; + if (options?.Tools is null || options.Tools.Count == 0) + { + return options; + } - // remove all instances of HostedImageGenerationTool and store the options from the last one - if (tool is HostedImageGenerationTool imageGenerationTool) + List? newTools = null; + var tools = options.Tools; + for (int i = 0; i < tools.Count; i++) { - _imageGenerationOptions = imageGenerationTool.Options; + var tool = tools[i]; - // for the first image generation tool, clone the options and insert our function tools - // remove any subsequent image generation tools - if (newTools is null) + // remove all instances of HostedImageGenerationTool and store the options from the last one + if (tool is HostedImageGenerationTool imageGenerationTool) { - newTools = CopyList(tools, i); - newTools.AddRange(_aiTools); + _imageGenerationOptions = imageGenerationTool.Options; + + // for the first image generation tool, clone the options and insert our function tools + // remove any subsequent image generation tools + if (newTools is null) + { + newTools = CopyList(tools, i); + newTools.AddRange(GetAITools()); + } + } + else + { + newTools?.Add(tool); } } - else + + if (newTools is not null) { - newTools?.Add(tool); + var newOptions = options.Clone(); + newOptions.Tools = newTools; + return newOptions; } - } - if (newTools is not null) - { - var newOptions = options.Clone(); - newOptions.Tools = newTools; - return newOptions; + return options; } - return options; - } - - private DataContent? RetrieveImageContent(string imageId) - { - if (_imageContentById.TryGetValue(imageId, out var imageContent)) + private DataContent? RetrieveImageContent(string imageId) { - return imageContent as DataContent; - } - - return null; - } - - private string StoreImage(DataContent imageContent, bool isGenerated = false) - { - // Generate a unique ID for the image if it doesn't have one - string? imageId = null; - if (imageContent.AdditionalProperties?.TryGetValue(ImageKey, out imageId) is false || imageId is null) - { - imageId = imageContent.Name ?? Guid.NewGuid().ToString(); - } + if (_imageContentById.TryGetValue(imageId, out var imageContent)) + { + return imageContent as DataContent; + } - if (isGenerated) - { - imageContent.AdditionalProperties ??= new(); - imageContent.AdditionalProperties[ImageKey] = imageId; + return null; } - // Store the image content for later retrieval - _imageContentById[imageId] = imageContent; - - return imageId; - } - - /// - /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. - /// We will have two messages - /// 1. Role: Assistant, FunctionCall - /// 2. Role: Tool, FunctionResult - /// We need to replace content from both but we shouldn't remove the messages. - /// If we do not then ChatClient's may not accept our altered history. - /// - /// When interating with a HostedImageGenerationTool we will have typically only see a single Message with - /// Role: Assistant that contains the DataContent (or a provider specific content, that's exposed as DataContent). - /// - /// The list of AI content to process. - private IList ReplaceImageGenerationFunctionResults(IList contents) - { - List? newContents = null; - - // Replace FunctionResultContent instances with generated image content - for (int i = contents.Count - 1; i >= 0; i--) + private string StoreImage(DataContent imageContent, bool isGenerated = false) { - var content = contents[i]; - - if (content is FunctionCallContent functionCall && - _functionNames.Contains(functionCall.Name)) + // Generate a unique ID for the image if it doesn't have one + string? imageId = null; + if (imageContent.AdditionalProperties?.TryGetValue(ImageKey, out imageId) is false || imageId is null) { - // create a new list and omit the FunctionCallContent - newContents ??= CopyList(contents, i); - - // add a placeholder text content to avoid empty contents which could cause the client to drop the message - newContents.Add(new TextContent(string.Empty)); + imageId = imageContent.Name ?? Guid.NewGuid().ToString(); } - else if (content is FunctionResultContent functionResult && - _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) - { - newContents ??= CopyList(contents, i, imageContents.Count - 1); - // Insert generated image content in its place, do not preserve the FunctionResultContent - foreach (var imageContent in imageContents) - { - newContents.Add(imageContent); - } - - // Remove the mapping as it's no longer needed - _ = _imageContentByCallId.Remove(functionResult.CallId); - } - else + if (isGenerated) { - // keep the existing content if we have a new list - newContents?.Add(content); + imageContent.AdditionalProperties ??= new(); + imageContent.AdditionalProperties[ImageKey] = imageId; } - } - return newContents ?? contents; - } + // Store the image content for later retrieval + _imageContentById[imageId] = imageContent; - [Description("Generates images based on a text description.")] - private async Task GenerateImageAsync( - [Description("A detailed description of the image to generate")] string prompt, - CancellationToken cancellationToken = default) - { - // Get the call ID from the current function invocation context - var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; - if (callId == null) - { - return "No call ID available for image generation."; + return imageId; } - var request = new ImageGenerationRequest(prompt); - var options = _imageGenerationOptions ?? new ImageGenerationOptions(); - options.Count ??= 1; - - var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); - - if (response.Contents.Count == 0) + /// + /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. + /// We will have two messages + /// 1. Role: Assistant, FunctionCall + /// 2. Role: Tool, FunctionResult + /// We need to replace content from both but we shouldn't remove the messages. + /// If we do not then ChatClient's may not accept our altered history. + /// + /// When interating with a HostedImageGenerationTool we will have typically only see a single Message with + /// Role: Assistant that contains the DataContent (or a provider specific content, that's exposed as DataContent). + /// + /// The list of AI content to process. + public IList ReplaceImageGenerationFunctionResults(IList contents) { - return "No image was generated."; - } + List? newContents = null; - List imageIds = []; - List imageContents = _imageContentByCallId[callId] = []; - foreach (var content in response.Contents) - { - if (content is DataContent imageContent && imageContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + // Replace FunctionResultContent instances with generated image content + for (int i = contents.Count - 1; i >= 0; i--) { - imageContents.Add(imageContent); - imageIds.Add(StoreImage(imageContent, true)); - } - } - - return "Generated image successfully."; - } + var content = contents[i]; - [Description("Lists the identifiers of all images available for edit.")] - private string[] GetImagesForEdit() - { - // Get the call ID from the current function invocation context - var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; - if (callId == null) - { - return ["No call ID available for image editing."]; - } + if (content is FunctionCallContent functionCall && + _imageContentByCallId.ContainsKey(functionCall.CallId)) + { + // create a new list and omit the FunctionCallContent + newContents ??= CopyList(contents, i); - _imageContentByCallId[callId] = []; + // add a placeholder text content to avoid empty contents which could cause the client to drop the message + newContents.Add(new TextContent(string.Empty)); + } + else if (content is FunctionResultContent functionResult && + _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) + { + newContents ??= CopyList(contents, i, imageContents.Count - 1); - return _imageContentById.Keys.ToArray(); - } + // Insert generated image content in its place, do not preserve the FunctionResultContent + foreach (var imageContent in imageContents) + { + newContents.Add(imageContent); + } - [Description("Edits an existing image based on a text description.")] - private async Task EditImageAsync( - [Description("A detailed description of the image to generate")] string prompt, - [Description($"The image to edit from one of the available image identifiers returned by {nameof(GetImagesForEdit)}")] string imageId, - CancellationToken cancellationToken = default) - { - // Get the call ID from the current function invocation context - var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; - if (callId == null) - { - return "No call ID available for image editing."; - } + // Remove the mapping as it's no longer needed + _ = _imageContentByCallId.Remove(functionResult.CallId); + } + else + { + // keep the existing content if we have a new list + newContents?.Add(content); + } + } - if (string.IsNullOrEmpty(imageId)) - { - return "No imageId provided"; + return newContents ?? contents; } - try + [Description("Generates images based on a text description.")] + public async Task GenerateImageAsync( + [Description("A detailed description of the image to generate")] string prompt, + CancellationToken cancellationToken = default) { - var originalImage = RetrieveImageContent(imageId); - if (originalImage == null) + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) { - return $"No image found with: {imageId}"; + return "No call ID available for image generation."; } - var request = new ImageGenerationRequest(prompt, [originalImage]); - var response = await _imageGenerator.GenerateAsync(request, _imageGenerationOptions, cancellationToken); + var request = new ImageGenerationRequest(prompt); + var options = _imageGenerationOptions ?? new ImageGenerationOptions(); + options.Count ??= 1; + + var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); if (response.Contents.Count == 0) { - return "No edited image was generated."; + return "No image was generated."; } List imageIds = []; @@ -467,11 +396,75 @@ private async Task EditImageAsync( } } - return "Edited image successfully."; + return "Generated image successfully."; + } + + [Description("Lists the identifiers of all images available for edit.")] + public string[] GetImagesForEdit() + { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return ["No call ID available for image editing."]; + } + + _imageContentByCallId[callId] = []; + + return _imageContentById.Keys.ToArray(); } - catch (FormatException) + + [Description("Edits an existing image based on a text description.")] + public async Task EditImageAsync( + [Description("A detailed description of the image to generate")] string prompt, + [Description($"The image to edit from one of the available image identifiers returned by {nameof(GetImagesForEdit)}")] string imageId, + CancellationToken cancellationToken = default) { - return "Invalid image data format. Please provide a valid base64-encoded image."; + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return "No call ID available for image editing."; + } + + if (string.IsNullOrEmpty(imageId)) + { + return "No imageId provided"; + } + + try + { + var originalImage = RetrieveImageContent(imageId); + if (originalImage == null) + { + return $"No image found with: {imageId}"; + } + + var request = new ImageGenerationRequest(prompt, [originalImage]); + var response = await _imageGenerator.GenerateAsync(request, _imageGenerationOptions, cancellationToken); + + if (response.Contents.Count == 0) + { + return "No edited image was generated."; + } + + List imageIds = []; + List imageContents = _imageContentByCallId[callId] = []; + foreach (var content in response.Contents) + { + if (content is DataContent imageContent && imageContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + imageContents.Add(imageContent); + imageIds.Add(StoreImage(imageContent, true)); + } + } + + return "Edited image successfully."; + } + catch (FormatException) + { + return "Invalid image data format. Please provide a valid base64-encoded image."; + } } } } From 9f9a430e68559ef49b8b0d017263f7129338a6ff Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 7 Oct 2025 10:34:58 -0700 Subject: [PATCH 22/24] Adjust prompt to improve integration test reliability --- .../ImageGeneratingChatClientIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs index 16724ba6003..ed405224415 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs @@ -197,7 +197,7 @@ public virtual async Task MultipleEdits_EditsLatestImage() chatHistory.AddRange(secondResponse.Messages); // Third: Second edit (should edit the latest version by default) - chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit that image to add birds")); + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit that last image to add birds")); var thirdResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); // Assert From 799a72e5f2b7b1744846bcd24dc536d30c9a8c6f Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 7 Oct 2025 13:15:01 -0700 Subject: [PATCH 23/24] Refactor tool initialization I verified that the tool creation is cached by ReflectionAIFunctionDescriptor This change includes a small optimization to avoid additional allocation around inserting tools into the options. --- .../ImageGeneratingChatClient.cs | 126 +++++++++--------- 1 file changed, 64 insertions(+), 62 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index c7a8c3cea02..df89eca6772 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -152,29 +152,6 @@ public RequestState(IImageGenerator imageGenerator, DataContentHandling dataCont _dataContentHandling = dataContentHandling; } - private static List CopyList(IList original, int toOffsetExclusive, int additionalCapacity = 0) - { - var newList = new List(original.Count + additionalCapacity); - - // Copy all items up to and excluding the current index - for (int j = 0; j < toOffsetExclusive; j++) - { - newList.Add(original[j]); - } - - return newList; - } - - private AITool[] GetAITools() - { - return - [ - AIFunctionFactory.Create(GenerateImageAsync), - AIFunctionFactory.Create(EditImageAsync), - AIFunctionFactory.Create(GetImagesForEdit) - ]; - } - public IEnumerable ProcessChatMessages(IEnumerable messages) { // If no special handling is needed, return the original messages @@ -216,10 +193,8 @@ public IEnumerable ProcessChatMessages(IEnumerable mes if (newContents != null) { - newMessages ??= new List(messages.Take(messageIndex)); - + newMessages ??= [.. messages.Take(messageIndex)]; var newMessage = message.Clone(); - newMessage.Contents = newContents; newMessages.Add(newMessage); } @@ -255,11 +230,7 @@ public IEnumerable ProcessChatMessages(IEnumerable mes // for the first image generation tool, clone the options and insert our function tools // remove any subsequent image generation tools - if (newTools is null) - { - newTools = CopyList(tools, i); - newTools.AddRange(GetAITools()); - } + newTools ??= InitializeTools(tools, i); } else { @@ -277,37 +248,6 @@ public IEnumerable ProcessChatMessages(IEnumerable mes return options; } - private DataContent? RetrieveImageContent(string imageId) - { - if (_imageContentById.TryGetValue(imageId, out var imageContent)) - { - return imageContent as DataContent; - } - - return null; - } - - private string StoreImage(DataContent imageContent, bool isGenerated = false) - { - // Generate a unique ID for the image if it doesn't have one - string? imageId = null; - if (imageContent.AdditionalProperties?.TryGetValue(ImageKey, out imageId) is false || imageId is null) - { - imageId = imageContent.Name ?? Guid.NewGuid().ToString(); - } - - if (isGenerated) - { - imageContent.AdditionalProperties ??= new(); - imageContent.AdditionalProperties[ImageKey] = imageId; - } - - // Store the image content for later retrieval - _imageContentById[imageId] = imageContent; - - return imageId; - } - /// /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. /// We will have two messages @@ -466,5 +406,67 @@ public async Task EditImageAsync( return "Invalid image data format. Please provide a valid base64-encoded image."; } } + + private static List CopyList(IList original, int toOffsetExclusive, int additionalCapacity = 0) + { + var newList = new List(original.Count + additionalCapacity); + + // Copy all items up to and excluding the current index + for (int j = 0; j < toOffsetExclusive; j++) + { + newList.Add(original[j]); + } + + return newList; + } + + private List InitializeTools(IList existingTools, int toOffsetExclusive) + { +#if NET + ReadOnlySpan tools = +#else + AITool[] tools = +#endif + [ + AIFunctionFactory.Create(GenerateImageAsync), + AIFunctionFactory.Create(EditImageAsync), + AIFunctionFactory.Create(GetImagesForEdit) + ]; + + var result = CopyList(existingTools, toOffsetExclusive, tools.Length); + result.AddRange(tools); + return result; + } + + private DataContent? RetrieveImageContent(string imageId) + { + if (_imageContentById.TryGetValue(imageId, out var imageContent)) + { + return imageContent as DataContent; + } + + return null; + } + + private string StoreImage(DataContent imageContent, bool isGenerated = false) + { + // Generate a unique ID for the image if it doesn't have one + string? imageId = null; + if (imageContent.AdditionalProperties?.TryGetValue(ImageKey, out imageId) is false || imageId is null) + { + imageId = imageContent.Name ?? Guid.NewGuid().ToString(); + } + + if (isGenerated) + { + imageContent.AdditionalProperties ??= []; + imageContent.AdditionalProperties[ImageKey] = imageId; + } + + // Store the image content for later retrieval + _imageContentById[imageId] = imageContent; + + return imageId; + } } } From 6029b018ca10fb8ec3f02c005ed3865ee622db86 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 7 Oct 2025 14:45:12 -0700 Subject: [PATCH 24/24] Add integration tests for streaming Fixes the removal of tool content - this was broken for streaming when I changed removal to be based on callId. We don't have the CallId yet in the streaming case so we have to remove by name. --- .../ImageGeneratingChatClient.cs | 9 +- ...ageGeneratingChatClientIntegrationTests.cs | 147 ++++++++++++------ 2 files changed, 110 insertions(+), 46 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index df89eca6772..ae36e5f8ecf 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -142,6 +142,7 @@ private sealed class RequestState { private readonly IImageGenerator _imageGenerator; private readonly DataContentHandling _dataContentHandling; + private readonly HashSet _toolNames = new(StringComparer.Ordinal); private readonly Dictionary> _imageContentByCallId = []; private readonly Dictionary _imageContentById = new(StringComparer.OrdinalIgnoreCase); private ImageGenerationOptions? _imageGenerationOptions; @@ -269,8 +270,9 @@ public IList ReplaceImageGenerationFunctionResults(IList c { var content = contents[i]; + // We must lookup by name because in the streaming case we have not yet been called to record the CallId. if (content is FunctionCallContent functionCall && - _imageContentByCallId.ContainsKey(functionCall.CallId)) + _toolNames.Contains(functionCall.Name)) { // create a new list and omit the FunctionCallContent newContents ??= CopyList(contents, i); @@ -433,6 +435,11 @@ private List InitializeTools(IList existingTools, int toOffsetEx AIFunctionFactory.Create(GetImagesForEdit) ]; + foreach (var tool in tools) + { + _toolNames.Add(tool.Name); + } + var result = CopyList(existingTools, toOffsetExclusive, tools.Length); result.AddRange(tools); return result; diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs index ed405224415..ca13fa88152 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs @@ -21,12 +21,13 @@ namespace Microsoft.Extensions.AI; /// public abstract class ImageGeneratingChatClientIntegrationTests : IDisposable { + private const string ImageKey = "meai_image"; private readonly IChatClient? _baseChatClient; protected ImageGeneratingChatClientIntegrationTests() { _baseChatClient = CreateChatClient(); - ImageGenerator = CreateImageGenerator(); + ImageGenerator = new(); if (_baseChatClient != null) { @@ -42,7 +43,7 @@ protected ImageGeneratingChatClientIntegrationTests() protected IChatClient? ChatClient { get; } /// Gets the IImageGenerator used for testing. - protected IImageGenerator ImageGenerator { get; } + protected CapturingImageGenerator ImageGenerator { get; } public void Dispose() { @@ -60,25 +61,66 @@ public void Dispose() protected abstract IChatClient? CreateChatClient(); /// - /// Creates the IImageGenerator implementation for testing. - /// The default implementation creates a test image generator that captures calls. + /// Helper method to get a chat response using either streaming or non-streaming based on the parameter. /// - /// An IImageGenerator instance for testing. - protected virtual IImageGenerator CreateImageGenerator() => new CapturingImageGenerator(); + /// Whether to use streaming or non-streaming response. + /// The chat messages to send. + /// The chat options to use. + /// A ChatResponse from either streaming or non-streaming call. + protected async Task GetResponseAsync(bool useStreaming, IEnumerable messages, ChatOptions? options = null, IChatClient? chatClient = null) + { + chatClient ??= ChatClient ?? throw new InvalidOperationException("ChatClient is not initialized."); + + if (useStreaming) + { + return ValidateChatResponse(await chatClient.GetStreamingResponseAsync(messages, options).ToChatResponseAsync()); + } + else + { + return ValidateChatResponse(await chatClient.GetResponseAsync(messages, options)); + } + + static ChatResponse ValidateChatResponse(ChatResponse response) + { + var contents = response.Messages.SelectMany(m => m.Contents).ToArray(); - [ConditionalFact] - public virtual async Task GenerateImage_CallsGenerateFunction_ReturnsDataContent() + List imageIds = []; + foreach (var dataContent in contents.OfType()) + { + var imageId = dataContent.AdditionalProperties?[ImageKey] as string; + Assert.NotNull(imageId); + imageIds.Add(imageId); + } + + foreach (var textContent in contents.OfType()) + { + Assert.DoesNotContain(ImageKey, textContent.Text, StringComparison.OrdinalIgnoreCase); + foreach (var imageId in imageIds) + { + // Ensure no image IDs appear in text content + Assert.DoesNotContain(imageId, textContent.Text, StringComparison.OrdinalIgnoreCase); + } + } + + return response; + } + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task GenerateImage_CallsGenerateFunction_ReturnsDataContent(bool useStreaming) { SkipIfNotEnabled(); - var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerator = ImageGenerator; var chatOptions = new ChatOptions { Tools = [new HostedImageGenerationTool()] }; // Act - var response = await ChatClient.GetResponseAsync( + var response = await GetResponseAsync(useStreaming, [new ChatMessage(ChatRole.User, "Please generate an image of a cat")], chatOptions); @@ -98,12 +140,14 @@ [new ChatMessage(ChatRole.User, "Please generate an image of a cat")], Assert.False(imageContent.Data.IsEmpty); } - [ConditionalFact] - public virtual async Task EditImage_WithImageInSameRequest_PassesExactDataContent() + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task EditImage_WithImageInSameRequest_PassesExactDataContent(bool useStreaming) { SkipIfNotEnabled(); - var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerator = ImageGenerator; var testImageData = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG header var originalImageData = new DataContent(testImageData, "image/png") { Name = "original.png" }; var chatOptions = new ChatOptions @@ -112,7 +156,7 @@ public virtual async Task EditImage_WithImageInSameRequest_PassesExactDataConten }; // Act - var response = await ChatClient.GetResponseAsync( + var response = await GetResponseAsync(useStreaming, [new ChatMessage(ChatRole.User, [new TextContent("Please edit this image to add a red border"), originalImageData])], chatOptions); @@ -127,12 +171,14 @@ [new ChatMessage(ChatRole.User, [new TextContent("Please edit this image to add Assert.Equal("original.png", originalImageContent.Name); } - [ConditionalFact] - public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage() + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage(bool useStreaming) { SkipIfNotEnabled(); - var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerator = ImageGenerator; var chatOptions = new ChatOptions { Tools = [new HostedImageGenerationTool()] @@ -144,12 +190,12 @@ public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage() }; // First request: Generate image - var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var firstResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); chatHistory.AddRange(firstResponse.Messages); // Second request: Edit the generated image chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to make it more colorful")); - var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var secondResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); // Assert Assert.Equal(2, imageGenerator.GenerateCalls.Count); @@ -171,12 +217,14 @@ public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage() Assert.Contains("generated_image_1", editedImage.Name); } - [ConditionalFact] - public virtual async Task MultipleEdits_EditsLatestImage() + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task MultipleEdits_EditsLatestImage(bool useStreaming) { SkipIfNotEnabled(); - var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerator = ImageGenerator; var chatOptions = new ChatOptions { Tools = [new HostedImageGenerationTool()] @@ -188,17 +236,17 @@ public virtual async Task MultipleEdits_EditsLatestImage() }; // First: Generate image - var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var firstResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); chatHistory.AddRange(firstResponse.Messages); // Second: First edit chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add flowers")); - var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var secondResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); chatHistory.AddRange(secondResponse.Messages); // Third: Second edit (should edit the latest version by default) chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit that last image to add birds")); - var thirdResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var thirdResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); // Assert Assert.Equal(3, imageGenerator.GenerateCalls.Count); @@ -211,12 +259,14 @@ public virtual async Task MultipleEdits_EditsLatestImage() Assert.Equal(secondImage, lastImageToEdit); } - [ConditionalFact] - public virtual async Task MultipleEdits_EditsFirstImage() + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task MultipleEdits_EditsFirstImage(bool useStreaming) { SkipIfNotEnabled(); - var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerator = ImageGenerator; var chatOptions = new ChatOptions { Tools = [new HostedImageGenerationTool()] @@ -228,17 +278,17 @@ public virtual async Task MultipleEdits_EditsFirstImage() }; // First: Generate image - var firstResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var firstResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); chatHistory.AddRange(firstResponse.Messages); // Second: First edit chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add fruit")); - var secondResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var secondResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); chatHistory.AddRange(secondResponse.Messages); // Third: Second edit (should edit the latest version by default) chatHistory.Add(new ChatMessage(ChatRole.User, "That didn't work out. Please edit the original image to add birds")); - var thirdResponse = await ChatClient.GetResponseAsync(chatHistory, chatOptions); + var thirdResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); // Assert Assert.Equal(3, imageGenerator.GenerateCalls.Count); @@ -251,12 +301,14 @@ public virtual async Task MultipleEdits_EditsFirstImage() Assert.Equal(firstGeneratedImage, lastImageToEdit); } - [ConditionalFact] - public virtual async Task ImageGeneration_WithOptions_PassesOptionsToGenerator() + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task ImageGeneration_WithOptions_PassesOptionsToGenerator(bool useStreaming) { SkipIfNotEnabled(); - var imageGenerator = (CapturingImageGenerator)ImageGenerator; + var imageGenerator = ImageGenerator; var imageGenerationOptions = new ImageGenerationOptions { Count = 2, @@ -269,7 +321,7 @@ public virtual async Task ImageGeneration_WithOptions_PassesOptionsToGenerator() }; // Act - var response = await ChatClient.GetResponseAsync( + var response = await GetResponseAsync(useStreaming, [new ChatMessage(ChatRole.User, "Generate an image of a castle")], chatOptions); @@ -281,8 +333,10 @@ [new ChatMessage(ChatRole.User, "Generate an image of a castle")], Assert.Equal(new System.Drawing.Size(512, 512), options.ImageSize); } - [ConditionalFact] - public virtual async Task ImageContentHandling_AllImages_ReplacesImagesWithPlaceholders() + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task ImageContentHandling_AllImages_ReplacesImagesWithPlaceholders(bool useStreaming) { SkipIfNotEnabled(); @@ -290,7 +344,6 @@ public virtual async Task ImageContentHandling_AllImages_ReplacesImagesWithPlace var capturedMessages = new List>(); // Create a new ImageGeneratingChatClient with AllImages data content handling - using var imageGeneratingClient = _baseChatClient! .AsBuilder() .UseImageGeneration(ImageGenerator) @@ -305,12 +358,16 @@ public virtual async Task ImageContentHandling_AllImages_ReplacesImagesWithPlace var originalImage = new DataContent(testImageData, "image/png") { Name = "test.png" }; // Act - await imageGeneratingClient.GetResponseAsync([ - new ChatMessage(ChatRole.User, [ - new TextContent("Here's an image to process"), - originalImage - ]) - ], new ChatOptions { Tools = [new HostedImageGenerationTool()] }); + await GetResponseAsync(useStreaming, + [ + new ChatMessage(ChatRole.User, + [ + new TextContent("Here's an image to process"), + originalImage + ]) + ], + new ChatOptions { Tools = [new HostedImageGenerationTool()] }, + imageGeneratingClient); // Assert Assert.NotEmpty(capturedMessages); @@ -319,7 +376,7 @@ await imageGeneratingClient.GetResponseAsync([ // Should have text content with placeholder instead of original image var textContents = userMessage.Contents.OfType().ToList(); - Assert.Contains(textContents, tc => tc.Text.Contains("[meai_image:") && tc.Text.Contains("] available for edit")); + Assert.Contains(textContents, tc => tc.Text.Contains(ImageKey) && tc.Text.Contains("] available for edit")); // Should not contain the original DataContent Assert.DoesNotContain(userMessage.Contents, c => c == originalImage);