Image generation tool

NuGet.config

@@ -17,6 +17,8 @@
     <add key="dotnet9-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9-transport/nuget/v3/index.json" />
     <!-- Used for the Rich Navigation indexing task -->
     <add key="richnav" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-buildservices/nuget/v3/index.json" />
+
+    <add key="ericstj-staging" value="https://www.myget.org/F/ericstj-staging/api/v3/index.json" />
   </packageSources>
   <!-- Define mappings by adding package patterns beneath the target source.
        https://aka.ms/nuget-package-source-mapping  -->
@@ -39,6 +41,9 @@
     <packageSource key="richnav">
       <package pattern="*" />
     </packageSource>
+    <packageSource key="ericstj-staging">
+      <package pattern="OpenAi" />
+    </packageSource>
   </packageSourceMapping>
   <disabledPackageSources>
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->

eng/packages/General.props

@@ -16,7 +16,7 @@
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="OllamaSharp" Version="5.1.9" />
-    <PackageVersion Include="OpenAI" Version="2.5.0" />
+    <PackageVersion Include="OpenAI" Version="2.6.0-preview1" />
     <PackageVersion Include="Polly" Version="8.4.2" />
     <PackageVersion Include="Polly.Core" Version="8.4.2" />
     <PackageVersion Include="Polly.Extensions" Version="8.4.2" />

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -119,6 +119,95 @@ static async Task AddMessagesAsync(
             list.AddMessages(await updates.ToChatResponseAsync(cancellationToken).ConfigureAwait(false));
     }
 
+    /// <summary>Applies a <see cref="ChatResponseUpdate"/> to an existing <see cref="ChatResponse"/>.</summary>
+    /// <param name="response">The response to which the update should be applied.</param>
+    /// <param name="update">The update to apply to the response.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="response"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentNullException"><paramref name="update"/> is <see langword="null"/>.</exception>
+    /// <remarks>
+    /// This method modifies the existing <paramref name="response"/> by incorporating the content and metadata
+    /// from the <paramref name="update"/>. This includes using <see cref="ChatResponseUpdate.MessageId"/> to determine
+    /// message boundaries, as well as coalescing contiguous <see cref="AIContent"/> items where applicable, e.g. multiple
+    /// <see cref="TextContent"/> instances in a row may be combined into a single <see cref="TextContent"/>.
+    /// </remarks>
+    [Experimental("MEAI0001")]
+    public static void ApplyUpdate(this ChatResponse response, ChatResponseUpdate update)
+    {
+        _ = Throw.IfNull(response);
+        _ = Throw.IfNull(update);
+
+        ProcessUpdate(update, response);
+        FinalizeResponse(response);
+    }
+
+    /// <summary>Applies <see cref="ChatResponseUpdate"/> instances to an existing <see cref="ChatResponse"/>.</summary>
+    /// <param name="response">The response to which the updates should be applied.</param>
+    /// <param name="updates">The updates to apply to the response.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="response"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentNullException"><paramref name="updates"/> is <see langword="null"/>.</exception>
+    /// <remarks>
+    /// This method modifies the existing <paramref name="response"/> by incorporating the content and metadata
+    /// from the <paramref name="updates"/>. This includes using <see cref="ChatResponseUpdate.MessageId"/> to determine
+    /// message boundaries, as well as coalescing contiguous <see cref="AIContent"/> items where applicable, e.g. multiple
+    /// <see cref="TextContent"/> instances in a row may be combined into a single <see cref="TextContent"/>.
+    /// </remarks>
+    [Experimental("MEAI0001")]
+    public static void ApplyUpdates(this ChatResponse response, IEnumerable<ChatResponseUpdate> updates)
+    {
+        _ = Throw.IfNull(response);
+        _ = Throw.IfNull(updates);
+
+        if (updates is ICollection<ChatResponseUpdate> { Count: 0 })
+        {
+            return;
+        }
+
+        foreach (var update in updates)
+        {
+            ProcessUpdate(update, response);
+        }
+
+        FinalizeResponse(response);
+    }
+
+    /// <summary>Applies <see cref="ChatResponseUpdate"/> instances to an existing <see cref="ChatResponse"/> asynchronously.</summary>
+    /// <param name="response">The response to which the updates should be applied.</param>
+    /// <param name="updates">The updates to apply to the response.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>A <see cref="Task"/> representing the completion of the operation.</returns>
+    /// <exception cref="ArgumentNullException"><paramref name="response"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentNullException"><paramref name="updates"/> is <see langword="null"/>.</exception>
+    /// <remarks>
+    /// This method modifies the existing <paramref name="response"/> by incorporating the content and metadata
+    /// from the <paramref name="updates"/>. This includes using <see cref="ChatResponseUpdate.MessageId"/> to determine
+    /// message boundaries, as well as coalescing contiguous <see cref="AIContent"/> items where applicable, e.g. multiple
+    /// <see cref="TextContent"/> instances in a row may be combined into a single <see cref="TextContent"/>.
+    /// </remarks>
+    [Experimental("MEAI0001")]
+    public static Task ApplyUpdatesAsync(
+        this ChatResponse response,
+        IAsyncEnumerable<ChatResponseUpdate> updates,
+        CancellationToken cancellationToken = default)
+    {
+        _ = Throw.IfNull(response);
+        _ = Throw.IfNull(updates);
+
+        return ApplyUpdatesAsync(response, updates, cancellationToken);
+
+        static async Task ApplyUpdatesAsync(
+            ChatResponse response,
+            IAsyncEnumerable<ChatResponseUpdate> updates,
+            CancellationToken cancellationToken)
+        {
+            await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false))
+            {
+                ProcessUpdate(update, response);
+            }
+
+            FinalizeResponse(response);
+        }
+    }
+
     /// <summary>Combines <see cref="ChatResponseUpdate"/> instances into a single <see cref="ChatResponse"/>.</summary>
     /// <param name="updates">The updates to be combined.</param>
     /// <returns>The combined <see cref="ChatResponse"/>.</returns>
@@ -372,6 +461,29 @@ private static void ProcessUpdate(ChatResponseUpdate update, ChatResponse respon
                     (response.Usage ??= new()).Add(usage.Details);
                     break;
 
+                case DataContent dataContent when
+                    !string.IsNullOrEmpty(dataContent.Name):
+                    // Check if there's an existing DataContent with the same name to replace
+                    for (int i = 0; i < message.Contents.Count; i++)
+                    {
+                        if (message.Contents[i] is DataContent existingDataContent &&
+                            string.Equals(existingDataContent.Name, dataContent.Name, StringComparison.Ordinal))
+                        {
+                            // Replace the existing DataContent
+                            message.Contents[i] = dataContent;
+                            dataContent = null!;
+                            break;
+                        }
+                    }
+
+                    if (dataContent is not null)
+                    {
+                        // No existing DataContent with the same name, add it normally
+                        message.Contents.Add(dataContent);
+                    }
+
+                    break;
+
                 default:
                     message.Contents.Add(content);
                     break;

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs

@@ -20,7 +20,7 @@ namespace Microsoft.Extensions.AI;
 /// </para>
 /// <para>
 /// The relationship between <see cref="ChatResponse"/> and <see cref="ChatResponseUpdate"/> is
-/// codified in the <see cref="ChatResponseExtensions.ToChatResponseAsync"/> and
+/// codified in the <see cref="ChatResponseExtensions.ToChatResponseAsync(IAsyncEnumerable{ChatResponseUpdate}, System.Threading.CancellationToken)"/> and
 /// <see cref="ChatResponse.ToChatResponseUpdates"/>, which enable bidirectional conversions
 /// between the two. Note, however, that the provided conversions may be lossy, for example if multiple
 /// updates all have different <see cref="RawRepresentation"/> objects whereas there's only one slot for
@@ -58,6 +58,30 @@ public ChatResponseUpdate(ChatRole? role, IList<AIContent>? contents)
         _contents = contents;
     }
 
+    /// <summary>
+    /// Creates a new ChatResponseUpdate instance that is a copy of the current object.
+    /// </summary>
+    /// <remarks>The cloned object is a shallow copy; reference-type properties will reference the same
+    /// objects as the original. Use this method to duplicate the response update for further modification without
+    /// affecting the original instance.</remarks>
+    /// <returns>A new ChatResponseUpdate object with the same property values as the current instance.</returns>
+    [Experimental("MEAI001")]
+    public ChatResponseUpdate Clone() =>
+        new()
+        {
+            AdditionalProperties = AdditionalProperties,
+            AuthorName = AuthorName,
+            Contents = Contents,
+            CreatedAt = CreatedAt,
+            ConversationId = ConversationId,
+            FinishReason = FinishReason,
+            MessageId = MessageId,
+            ModelId = ModelId,
+            RawRepresentation = RawRepresentation,
+            ResponseId = ResponseId,
+            Role = Role,
+        };
+
     /// <summary>Gets or sets the name of the author of the response update.</summary>
     public string? AuthorName
     {

src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Represents a hosted tool that can be specified to an AI service to enable it to perform image generation.</summary>
+/// <remarks>
+/// This tool does not itself implement image generation. It is a marker that can be used to inform a service
+/// that the service is allowed to perform image generation if the service is capable of doing so.
+/// </remarks>
+[Experimental("MEAI001")]
+public class HostedImageGenerationTool : AITool
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="HostedImageGenerationTool"/> class with the specified options.
+    /// </summary>
+    public HostedImageGenerationTool()
+        : base()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the options used to configure image generation.
+    /// </summary>
+    public ImageGenerationOptions? Options { get; set; }
+
+    /// <summary>
+    /// Gets or sets a callback responsible for creating the raw representation of the image generation tool from an underlying implementation.
+    /// </summary>
+    /// <remarks>
+    /// The underlying <see cref="IChatClient" /> implementation can have its own representation of this tool.
+    /// When <see cref="IChatClient.GetResponseAsync" /> or <see cref="IChatClient.GetStreamingResponseAsync" /> is invoked with an
+    /// <see cref="HostedImageGenerationTool" />, that implementation can convert the provided tool and options into its own representation in
+    /// order to use it while performing the operation. For situations where a consumer knows which concrete <see cref="IChatClient" /> is being used
+    /// and how it represents this tool, a new instance of that implementation-specific tool type can be returned by this
+    /// callback for the <see cref="IChatClient" /> implementation to use instead of creating a new instance.
+    /// Such implementations might mutate the supplied options instance further based on other settings supplied on this
+    /// <see cref="HostedImageGenerationTool" /> instance or from other inputs, therefore, it is <b>strongly recommended</b> to not
+    /// return shared instances and instead make the callback return a new instance on each call.
+    /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly typed
+    /// properties on <see cref="ImageGenerationOptions" />.
+    /// </remarks>
+    [JsonIgnore]
+    public Func<IChatClient, object?>? RawRepresentationFactory { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs

@@ -93,6 +93,7 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(IEnumerable<string>))]
     [JsonSerializable(typeof(char))]
     [JsonSerializable(typeof(string))]
+    [JsonSerializable(typeof(string[]))]
     [JsonSerializable(typeof(int))]
     [JsonSerializable(typeof(short))]
     [JsonSerializable(typeof(long))]

src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj

@@ -27,7 +27,6 @@
     <InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
     <InjectRequiredMemberOnLegacy>true</InjectRequiredMemberOnLegacy>
     <InjectSharedEmptyCollections>true</InjectSharedEmptyCollections>
-    <InjectSharedServerSentEvents>true</InjectSharedServerSentEvents>
     <InjectStringHashOnLegacy>true</InjectStringHashOnLegacy>
   </PropertyGroup>
 

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs

@@ -14,8 +14,10 @@ namespace Microsoft.Extensions.AI;
     WriteIndented = true)]
 [JsonSerializable(typeof(OpenAIClientExtensions.ToolJson))]
 [JsonSerializable(typeof(IDictionary<string, object?>))]
+[JsonSerializable(typeof(IDictionary<string, string?>))]
 [JsonSerializable(typeof(IReadOnlyDictionary<string, object?>))]
 [JsonSerializable(typeof(string[]))]
 [JsonSerializable(typeof(IEnumerable<string>))]
 [JsonSerializable(typeof(JsonElement))]
+[JsonSerializable(typeof(int))]
 internal sealed partial class OpenAIJsonContext : JsonSerializerContext;