Remove ImageContent and AudioContent

eng/MSBuild/Shared.props

@@ -14,7 +14,7 @@
   <ItemGroup Condition="'$(InjectSharedRentedSpan)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\RentedSpan\*.cs" LinkBase="Shared\RentedSpan" />
   </ItemGroup>
-  
+
   <ItemGroup Condition="'$(InjectSharedServerSentEvents)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\ServerSentEvents\*.cs" LinkBase="Shared\ServerSentEvents" />
   </ItemGroup>
@@ -46,4 +46,8 @@
   <ItemGroup Condition="'$(InjectStringSplitExtensions)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\StringSplit\*.cs" LinkBase="Shared\StringSplit" />
   </ItemGroup>
+
+  <ItemGroup Condition="'$(InjectSharedAIExtensions)' == 'true'">
+    <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\AIExtensions\*.cs" LinkBase="Shared\AIExtensions" />
+  </ItemGroup>
 </Project>

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -7,11 +7,9 @@ namespace Microsoft.Extensions.AI;
 
 /// <summary>Provides a base class for all content used with AI services.</summary>
 [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
-[JsonDerivedType(typeof(AudioContent), typeDiscriminator: "audio")]
 [JsonDerivedType(typeof(DataContent), typeDiscriminator: "data")]
 [JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: "functionCall")]
 [JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: "functionResult")]
-[JsonDerivedType(typeof(ImageContent), typeDiscriminator: "image")]
 [JsonDerivedType(typeof(TextContent), typeDiscriminator: "text")]
 [JsonDerivedType(typeof(UsageContent), typeDiscriminator: "usage")]
 public class AIContent

src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs

@@ -484,12 +484,16 @@ private static List<ChatMessageContentItem> GetContentParts(IList<AIContent> con
                     parts.Add(new ChatMessageTextContentItem(textContent.Text));
                     break;
 
-                case ImageContent imageContent when imageContent.Data is { IsEmpty: false } data:
-                    parts.Add(new ChatMessageImageContentItem(BinaryData.FromBytes(data), imageContent.MediaType));
-                    break;
+                case DataContent dataContent when dataContent.HasImageMediaType():
+                    if (dataContent.Data is { IsEmpty: false } data)
+                    {
+                        parts.Add(new ChatMessageImageContentItem(BinaryData.FromBytes(data), dataContent.MediaType));
+                    }
+                    else if (dataContent.Uri is string uri)
+                    {
+                        parts.Add(new ChatMessageImageContentItem(new Uri(uri)));
+                    }
 
-                case ImageContent imageContent when imageContent.Uri is string uri:
-                    parts.Add(new ChatMessageImageContentItem(new Uri(uri)));
                     break;
             }
         }

src/Libraries/Microsoft.Extensions.AI.AzureAIInference/Microsoft.Extensions.AI.AzureAIInference.csproj

@@ -25,8 +25,9 @@
     <InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
     <InjectSharedEmptyCollections>true</InjectSharedEmptyCollections>
     <InjectStringHashOnLegacy>true</InjectStringHashOnLegacy>
+    <InjectSharedAIExtensions>true</InjectSharedAIExtensions>
   </PropertyGroup>
-  
+
   <ItemGroup>
     <PackageReference Include="Azure.AI.Inference" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
@@ -37,5 +38,5 @@
   <ItemGroup>
     <ProjectReference Include="../Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
-  
+
 </Project>

src/Libraries/Microsoft.Extensions.AI.Ollama/Microsoft.Extensions.AI.Ollama.csproj

@@ -26,6 +26,7 @@
     <InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
     <InjectSharedEmptyCollections>true</InjectSharedEmptyCollections>
     <InjectStringHashOnLegacy>true</InjectStringHashOnLegacy>
+    <InjectSharedAIExtensions>true</InjectSharedAIExtensions>
   </PropertyGroup>
 
   <ItemGroup>
@@ -40,5 +41,5 @@
   <ItemGroup>
     <ProjectReference Include="../Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
-  
+
 </Project>

src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs

@@ -375,7 +375,7 @@ private IEnumerable<OllamaChatRequestMessage> ToOllamaChatRequestMessages(ChatMe
         OllamaChatRequestMessage? currentTextMessage = null;
         foreach (var item in content.Contents)
         {
-            if (currentTextMessage is not null && item is not ImageContent)
+            if (currentTextMessage is not null && item is not DataContent)
             {
                 yield return currentTextMessage;
                 currentTextMessage = null;
@@ -391,9 +391,9 @@ private IEnumerable<OllamaChatRequestMessage> ToOllamaChatRequestMessages(ChatMe
                     };
                     break;
 
-                case ImageContent imageContent when imageContent.Data is not null:
+                case DataContent dataContent when dataContent.Data is not null && dataContent.HasImageMediaType():
                     IList<string> images = currentTextMessage?.Images ?? [];
-                    images.Add(Convert.ToBase64String(imageContent.Data.Value
+                    images.Add(Convert.ToBase64String(dataContent.Data.Value
 #if NET
                         .Span));
 #else

src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj

@@ -27,6 +27,7 @@
     <InjectSharedServerSentEvents>true</InjectSharedServerSentEvents>
     <InjectRequiredMemberOnLegacy>true</InjectRequiredMemberOnLegacy>
     <InjectCompilerFeatureRequiredOnLegacy>true</InjectCompilerFeatureRequiredOnLegacy>
+    <InjectSharedAIExtensions>true</InjectSharedAIExtensions>
   </PropertyGroup>
 
   <ItemGroup>
@@ -38,5 +39,5 @@
   <ItemGroup>
     <ProjectReference Include="../Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
-  
+
 </Project>

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatCompletion.cs

@@ -536,10 +536,10 @@ private static ChatRole FromOpenAIChatRole(ChatMessageRole role) =>
         }
         else if (contentPart.Kind == ChatMessageContentPartKind.Image)
         {
-            ImageContent? imageContent;
+            DataContent? imageContent;
             aiContent = imageContent =
-                contentPart.ImageUri is not null ? new ImageContent(contentPart.ImageUri, contentPart.ImageBytesMediaType) :
-                contentPart.ImageBytes is not null ? new ImageContent(contentPart.ImageBytes.ToMemory(), contentPart.ImageBytesMediaType) :
+                contentPart.ImageUri is not null ? new DataContent(contentPart.ImageUri, contentPart.ImageBytesMediaType) :
+                contentPart.ImageBytes is not null ? new DataContent(contentPart.ImageBytes.ToMemory(), contentPart.ImageBytesMediaType) :
                 null;
 
             if (imageContent is not null && contentPart.ImageDetailLevel?.ToString() is string detail)

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatMessage.cs

@@ -185,7 +185,7 @@ public static IEnumerable<ChatMessage> FromOpenAIChatMessages(IEnumerable<OpenAI
 
     private static List<AIContent> FromOpenAIChatContent(IList<ChatMessageContentPart> openAiMessageContentParts)
     {
-        List<AIContent> contents = new();
+        List<AIContent> contents = [];
         foreach (var openAiContentPart in openAiMessageContentParts)
         {
             switch (openAiContentPart.Kind)
@@ -194,14 +194,13 @@ private static List<AIContent> FromOpenAIChatContent(IList<ChatMessageContentPar
                     contents.Add(new TextContent(openAiContentPart.Text));
                     break;
 
-                case ChatMessageContentPartKind.Image when (openAiContentPart.ImageBytes is { } bytes):
-                    contents.Add(new ImageContent(bytes.ToArray(), openAiContentPart.ImageBytesMediaType));
+                case ChatMessageContentPartKind.Image when openAiContentPart.ImageBytes is { } bytes:
+                    contents.Add(new DataContent(bytes.ToArray(), openAiContentPart.ImageBytesMediaType));
                     break;
 
                 case ChatMessageContentPartKind.Image:
-                    contents.Add(new ImageContent(openAiContentPart.ImageUri?.ToString() ?? string.Empty));
+                    contents.Add(new DataContent(openAiContentPart.ImageUri?.ToString() ?? string.Empty));
                     break;
-
             }
         }
 
@@ -220,12 +219,16 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
                     parts.Add(ChatMessageContentPart.CreateTextPart(textContent.Text));
                     break;
 
-                case ImageContent imageContent when imageContent.Data is { IsEmpty: false } data:
-                    parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(data), imageContent.MediaType));
-                    break;
+                case DataContent dataContent when dataContent.HasImageMediaType():
+                    if (dataContent.Data is { IsEmpty: false } data)
+                    {
+                        parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(data), dataContent.MediaType));
+                    }
+                    else if (dataContent.Uri is string uri)
+                    {
+                        parts.Add(ChatMessageContentPart.CreateImagePart(new Uri(uri)));
+                    }
 
-                case ImageContent imageContent when imageContent.Uri is string uri:
-                    parts.Add(ChatMessageContentPart.CreateImagePart(new Uri(uri)));
                     break;
             }
         }

src/Shared/AIExtensions/DataContentMediaTypeExtensions.cs

@@ -0,0 +1,23 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Provides extension methods for interpreting the media type of a <see cref="DataContent"/>.
+/// </summary>
+#if !SHARED_PROJECT
+[System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage]
+#endif
+internal static class DataContentMediaTypeExtensions
+{
+    /// <summary>
+    /// Determines whether the <see cref="DataContent"/> represents an image.
+    /// </summary>
+    /// <param name="content">The <see cref="DataContent"/>.</param>
+    /// <returns><see langword="true"/> if the content represents an image; otherwise, <see langword="false"/>.</returns>
+    public static bool HasImageMediaType(this DataContent content)
+        => content.MediaType is { } mediaType && mediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase);
+}

src/Shared/Shared.csproj

@@ -38,6 +38,10 @@
     <PackageReference Include="System.Text.Json" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\Libraries\Microsoft.Extensions.AI.Abstractions\Microsoft.Extensions.AI.Abstractions.csproj" />
+  </ItemGroup>
+
   <ItemGroup>
     <InternalsVisibleToTest Include="$(AssemblyName).Tests" />
     <InternalsVisibleToDynamicProxyGenAssembly2 Include="*" />

test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatCompletionTests.cs

@@ -239,7 +239,7 @@ public void ToStreamingChatCompletionUpdates_MultiChoice()
             new ChatMessage(ChatRole.Assistant,
             [
                 new TextContent("Hello, "),
-                new ImageContent("http://localhost/image.png"),
+                new DataContent("http://localhost/image.png", mediaType: "image/png"),
                 new TextContent("world!"),
             ])
             {
@@ -275,7 +275,7 @@ public void ToStreamingChatCompletionUpdates_MultiChoice()
         Assert.Equal(new DateTimeOffset(2024, 11, 10, 9, 20, 0, TimeSpan.Zero), update0.CreatedAt);
         Assert.Equal("assistant", update0.Role?.Value);
         Assert.Equal("Hello, ", Assert.IsType<TextContent>(update0.Contents[0]).Text);
-        Assert.IsType<ImageContent>(update0.Contents[1]);
+        Assert.Equal("image/png", Assert.IsType<DataContent>(update0.Contents[1]).MediaType);
         Assert.Equal("world!", Assert.IsType<TextContent>(update0.Contents[2]).Text);
         Assert.Equal("choice1Value", update0.AdditionalProperties?["choice1Key"]);