file download support

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs

@@ -141,12 +141,19 @@ public async Task<string> SayCommandAsync([ActionTurnContext] ITurnContext turnC
                 entity.Citation = referencedCitations;
             }
 
+            List<Attachment>? attachments = new();
+            if (command.Response.Attachments != null)
+            {
+                attachments = command.Response.Attachments;
+            }
+
             await turnContext.SendActivityAsync(new Activity()
             {
                 Type = ActivityTypes.Message,
                 Text = contentText,
                 ChannelData = channelData,
-                Entities = new List<Entity>() { entity }
+                Entities = new List<Entity>() { entity },
+                Attachments = attachments
             }, cancellationToken);
 
             return string.Empty;

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/AssistantsMessage.cs

@@ -1,4 +1,8 @@
-using OpenAI.Assistants;
+using Microsoft.Bot.Builder;
+using Microsoft.Bot.Schema;
+using OpenAI.Assistants;
+using OpenAI.Files;
+
 
 namespace Microsoft.Teams.AI.AI.Models
 {
@@ -12,43 +16,138 @@ public class AssistantsMessage : ChatMessage
         /// </summary>
         public MessageContent MessageContent;
 
+        /// <summary>
+        /// Files attached to the assistants api message.
+        /// </summary>
+        public List<OpenAIFile>? AttachedFiles { get; }
+
         /// <summary>
         /// Creates an AssistantMessage.
         /// </summary>
         /// <param name="content">The Assistants API thread message.</param>
-        public AssistantsMessage(MessageContent content) : base(ChatRole.Assistant)
+        /// <param name="attachedFiles">Generated files attached to the message.</param>
+        public AssistantsMessage(MessageContent content, List<OpenAIFile>? attachedFiles = null) : base(ChatRole.Assistant)
         {
             this.MessageContent = content;
 
-            if (content != null)
+            if (attachedFiles != null)
             {
-                string textContent = content.Text ?? "";
+                AttachedFiles = attachedFiles;
+                Attachments = _ConvertAttachedImagesToActivityAttachments(attachedFiles);
+            }
+        }
+
+        private List<Attachment> _ConvertAttachedImagesToActivityAttachments(List<OpenAIFile> attachedFiles)
+        {
+            List<Attachment> attachments = new();
+
+            foreach (OpenAIFile file in attachedFiles)
+            {
+                string? mimetype = file.GetMimeType();
+                string[] imageMimeTypes = new string[] { "image/png", "image/jpg", "image/jpeg", "image/gif" };
+                if (mimetype == null)
+                {
+                    continue;
+                }
 
-                MessageContext context = new();
-                for (int i = 0; i < content.TextAnnotations.Count; i++)
+                if (!imageMimeTypes.Contains(mimetype))
                 {
-                    TextAnnotation annotation = content.TextAnnotations[i];
-                    if (annotation?.TextToReplace != null)
-                    {
-                        textContent = textContent.Replace(annotation.TextToReplace, $"[{i + 1}]");
-                    }
-
-                    if (annotation?.InputFileId != null)
-                    {
-                        // Retrieve file info object
-                        // Neither `content` or `title` is provided in the annotations.
-                        context.Citations.Add(new("Content not available", $"File {i + 1}", annotation.InputFileId));
-                    }
-
-                    if (annotation?.OutputFileId != null)
-                    {
-                        // TODO: Download files or provide link to end user.
-                        // Files were generated by code interpretor tool.
-                    }
+                    // Skip non image file types
+                    continue;
                 }
 
-                Content = textContent;
-                Context = context;
+                string imageBase64String = Convert.ToBase64String(file.FileContent.ToArray());
+                attachments.Add(new Attachment
+                {
+                    Name = file.FileInfo.Filename,
+                    ContentType = mimetype,
+                    ContentUrl = $"data:image/png;base64,{imageBase64String}",
+                });
+            }
+
+            return attachments;
+        }
+    }
+
+    /// <summary>
+    /// Represents an OpenAI File.
+    /// </summary>
+    public class OpenAIFile
+    {
+        /// <summary>
+        /// Represents an OpenAI File information
+        /// </summary>
+        public OpenAIFileInfo FileInfo;
+
+        /// <summary>
+        /// Represents the contents of an OpenAI File
+        /// </summary>
+        public BinaryData FileContent;
+
+        private static readonly Dictionary<string, string> MimeTypes = new(StringComparer.OrdinalIgnoreCase)
+        {
+            { "c", "text/x-c" },
+            { "cs", "text/x-csharp" },
+            { "cpp", "text/x-c++" },
+            { "doc", "application/msword" },
+            { "docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document" },
+            { "html", "text/html" },
+            { "java", "text/x-java" },
+            { "json", "application/json" },
+            { "md", "text/markdown" },
+            { "pdf", "application/pdf" },
+            { "php", "text/x-php" },
+            { "pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" },
+            { "py", "text/x-python" },
+            { "rb", "text/x-ruby" },
+            { "tex", "text/x-tex" },
+            { "txt", "text/plain" },
+            { "css", "text/css" },
+            { "js", "text/javascript" },
+            { "sh", "application/x-sh" },
+            { "ts", "application/typescript" },
+            { "csv", "application/csv" },
+            { "jpeg", "image/jpeg" },
+            { "jpg", "image/jpeg" },
+            { "gif", "image/gif" },
+            { "png", "image/png" },
+            { "tar", "application/x-tar" },
+            { "xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" },
+            { "xml", "application/xml" }, // or "text/xml"
+            { "zip", "application/zip" }
+        };
+
+        /// <summary>
+        /// Initializes an instance of OpenAIFile
+        /// </summary>
+        /// <param name="fileInfo">The OpenAI File</param>
+        /// <param name="fileContent">The OpenAI File contents</param>
+        public OpenAIFile(OpenAIFileInfo fileInfo, BinaryData fileContent)
+        {
+            FileInfo = fileInfo;
+            FileContent = fileContent;
+        }
+
+        /// <summary>
+        /// Gets the file's mime type
+        /// </summary>
+        /// <returns>The file's mime type</returns>
+        public string? GetMimeType()
+        {
+            bool hasExtension = FileInfo.Filename.Contains(".");
+            if (!hasExtension)
+            {
+                return null;
+            }
+
+            string fileExtension = FileInfo.Filename.Split(new char[] { '.' }).Last();
+            if (MimeTypes.TryGetValue(fileExtension, out string mimeType))
+            {
+                return mimeType;
+            }
+            else
+            {
+                return null;
             }
         }
     }

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/ChatMessage.cs

@@ -1,5 +1,6 @@
 using Azure.AI.OpenAI;
 using Azure.AI.OpenAI.Chat;
+using Microsoft.Bot.Schema;
 using Microsoft.Teams.AI.Exceptions;
 using Microsoft.Teams.AI.Utilities;
 using OpenAI.Chat;
@@ -49,6 +50,10 @@ public class ChatMessage
         /// </summary>
         public IList<ChatCompletionsToolCall>? ToolCalls { get; set; }
 
+        /// <summary>
+        /// Attachments for the bot to send back.
+        /// </summary>
+        public List<Attachment>? Attachments { get; set; }
 
         /// <summary>
         /// Gets the content with the given type.

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/AssistantsPlanner.cs

@@ -58,7 +58,7 @@ public AssistantsPlanner(AssistantsPlannerOptions options, ILoggerFactory? logge
             {
                 Verify.ParamNotNull(options.Endpoint, "AssistantsPlannerOptions.Endpoint");
                 _client = _CreateClient(options.TokenCredential, options.Endpoint!);
-                _fileClient = _CreateFilesClient(options.TokenCredential, options.Endpoint!);
+                _fileClient = _CreateFileClient(options.TokenCredential, options.Endpoint!);
             }
             else if (options.ApiKey != null)
             {
@@ -233,7 +233,7 @@ private async Task<Plan> _GeneratePlanFromMessagesAsync(string threadId, string
             {
                 foreach (MessageContent content in message.Content)
                 {
-                    ChatMessage chatMessage = new AssistantsMessage(content);
+                    ChatMessage chatMessage = await _CreateAssistantMessageAsync(content);
                     plan.Commands.Add(new PredictedSayCommand(chatMessage));
                 }
             }
@@ -397,7 +397,7 @@ internal FileClient _CreateFileClient(string apiKey, string? endpoint = null)
             }
         }
 
-        internal FileClient _CreateFilesClient(TokenCredential tokenCredential, string endpoint)
+        internal FileClient _CreateFileClient(TokenCredential tokenCredential, string endpoint)
         {
             Verify.ParamNotNull(tokenCredential);
             Verify.ParamNotNull(endpoint);
@@ -441,6 +441,64 @@ private async Task<ThreadMessage> _CreateUserThreadMessageAsync(string threadId,
 
             return await _client.CreateMessageAsync(threadId, MessageRole.User, messages, options, cancellationToken);
         }
+
+        private async Task<AssistantsMessage> _CreateAssistantMessageAsync(MessageContent messageContent, CancellationToken cancellationToken = default)
+        {
+            if (messageContent == null)
+            {
+                throw new ArgumentNullException(nameof(messageContent));
+            }
+
+            string textContent = messageContent.Text ?? "";
+            MessageContext context = new();
+
+            List<Task<ClientResult<BinaryData>>> fileContentDownloadTasks = new();
+            List<Task<ClientResult<OpenAIFileInfo>>> fileInfoDownloadTasks = new();
+
+            for (int i = 0; i < messageContent.TextAnnotations.Count; i++)
+            {
+                TextAnnotation annotation = messageContent.TextAnnotations[i];
+                if (annotation?.TextToReplace != null)
+                {
+                    textContent = textContent.Replace(annotation.TextToReplace, $"[{i + 1}]");
+                }
+
+                if (annotation?.InputFileId != null)
+                {
+                    // Retrieve file info object
+                    // Neither `content` or `title` is provided in the annotations.
+                    context.Citations.Add(new("Content not available", $"File {i + 1}", annotation.InputFileId));
+                }
+
+                if (annotation?.OutputFileId != null)
+                {
+                    // Files generated by code interpretor tool.
+                    fileContentDownloadTasks.Add(_fileClient.DownloadFileAsync(annotation.OutputFileId));
+                    fileInfoDownloadTasks.Add(_fileClient.GetFileAsync(annotation.OutputFileId));
+                }
+            }
+
+            List<OpenAIFile> attachedFiles = new();
+            if (fileContentDownloadTasks.Count > 0)
+            {
+                // Create attachments out of these downloaded files
+                // Wait for tasks to complete
+                ClientResult<BinaryData>[] downloadedFileContent = await Task.WhenAll(fileContentDownloadTasks);
+                ClientResult<OpenAIFileInfo>[] downloadedFileInfo = await Task.WhenAll(fileInfoDownloadTasks);
+
+                for (int i = 0; i < downloadedFileContent.Length; i++)
+                {
+                    attachedFiles.Add(new OpenAIFile(downloadedFileInfo[i], downloadedFileContent[i]));
+                }
+            }
+
+            AssistantsMessage message = new(messageContent, attachedFiles);
+
+            message.Content = textContent;
+            message.Context = context;
+
+            return message;
+        }
     }
 }
 #pragma warning restore OPENAI001