From 0ef2fa4438eb373ecde9cf6f40f0dd839c35c3f9 Mon Sep 17 00:00:00 2001 From: Kavin Singh Date: Fri, 23 Aug 2024 09:18:53 -0700 Subject: [PATCH] file download support --- .../AI/Action/DefaultActions.cs | 9 +- .../AI/Models/AssistantsMessage.cs | 151 +++++++++++++++--- .../AI/Models/ChatMessage.cs | 5 + .../AI/Planners/AssistantsPlanner.cs | 64 +++++++- 4 files changed, 199 insertions(+), 30 deletions(-) diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs index c82653b4d..903dc14d6 100644 --- a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs +++ b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs @@ -141,12 +141,19 @@ public async Task SayCommandAsync([ActionTurnContext] ITurnContext turnC entity.Citation = referencedCitations; } + List? attachments = new(); + if (command.Response.Attachments != null) + { + attachments = command.Response.Attachments; + } + await turnContext.SendActivityAsync(new Activity() { Type = ActivityTypes.Message, Text = contentText, ChannelData = channelData, - Entities = new List() { entity } + Entities = new List() { entity }, + Attachments = attachments }, cancellationToken); return string.Empty; diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/AssistantsMessage.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/AssistantsMessage.cs index aad86877f..68d116323 100644 --- a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/AssistantsMessage.cs +++ b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/AssistantsMessage.cs @@ -1,4 +1,8 @@ -using OpenAI.Assistants; +using Microsoft.Bot.Builder; +using Microsoft.Bot.Schema; +using OpenAI.Assistants; +using OpenAI.Files; + namespace Microsoft.Teams.AI.AI.Models { @@ -12,43 +16,138 @@ public class AssistantsMessage : ChatMessage /// public MessageContent MessageContent; + /// + /// Files attached to the assistants api message. + /// + public List? AttachedFiles { get; } + /// /// Creates an AssistantMessage. /// /// The Assistants API thread message. - public AssistantsMessage(MessageContent content) : base(ChatRole.Assistant) + /// Generated files attached to the message. + public AssistantsMessage(MessageContent content, List? attachedFiles = null) : base(ChatRole.Assistant) { this.MessageContent = content; - if (content != null) + if (attachedFiles != null) { - string textContent = content.Text ?? ""; + AttachedFiles = attachedFiles; + Attachments = _ConvertAttachedImagesToActivityAttachments(attachedFiles); + } + } + + private List _ConvertAttachedImagesToActivityAttachments(List attachedFiles) + { + List attachments = new(); + + foreach (OpenAIFile file in attachedFiles) + { + string? mimetype = file.GetMimeType(); + string[] imageMimeTypes = new string[] { "image/png", "image/jpg", "image/jpeg", "image/gif" }; + if (mimetype == null) + { + continue; + } - MessageContext context = new(); - for (int i = 0; i < content.TextAnnotations.Count; i++) + if (!imageMimeTypes.Contains(mimetype)) { - TextAnnotation annotation = content.TextAnnotations[i]; - if (annotation?.TextToReplace != null) - { - textContent = textContent.Replace(annotation.TextToReplace, $"[{i + 1}]"); - } - - if (annotation?.InputFileId != null) - { - // Retrieve file info object - // Neither `content` or `title` is provided in the annotations. - context.Citations.Add(new("Content not available", $"File {i + 1}", annotation.InputFileId)); - } - - if (annotation?.OutputFileId != null) - { - // TODO: Download files or provide link to end user. - // Files were generated by code interpretor tool. - } + // Skip non image file types + continue; } - Content = textContent; - Context = context; + string imageBase64String = Convert.ToBase64String(file.FileContent.ToArray()); + attachments.Add(new Attachment + { + Name = file.FileInfo.Filename, + ContentType = mimetype, + ContentUrl = $"data:image/png;base64,{imageBase64String}", + }); + } + + return attachments; + } + } + + /// + /// Represents an OpenAI File. + /// + public class OpenAIFile + { + /// + /// Represents an OpenAI File information + /// + public OpenAIFileInfo FileInfo; + + /// + /// Represents the contents of an OpenAI File + /// + public BinaryData FileContent; + + private static readonly Dictionary MimeTypes = new(StringComparer.OrdinalIgnoreCase) + { + { "c", "text/x-c" }, + { "cs", "text/x-csharp" }, + { "cpp", "text/x-c++" }, + { "doc", "application/msword" }, + { "docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document" }, + { "html", "text/html" }, + { "java", "text/x-java" }, + { "json", "application/json" }, + { "md", "text/markdown" }, + { "pdf", "application/pdf" }, + { "php", "text/x-php" }, + { "pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" }, + { "py", "text/x-python" }, + { "rb", "text/x-ruby" }, + { "tex", "text/x-tex" }, + { "txt", "text/plain" }, + { "css", "text/css" }, + { "js", "text/javascript" }, + { "sh", "application/x-sh" }, + { "ts", "application/typescript" }, + { "csv", "application/csv" }, + { "jpeg", "image/jpeg" }, + { "jpg", "image/jpeg" }, + { "gif", "image/gif" }, + { "png", "image/png" }, + { "tar", "application/x-tar" }, + { "xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" }, + { "xml", "application/xml" }, // or "text/xml" + { "zip", "application/zip" } + }; + + /// + /// Initializes an instance of OpenAIFile + /// + /// The OpenAI File + /// The OpenAI File contents + public OpenAIFile(OpenAIFileInfo fileInfo, BinaryData fileContent) + { + FileInfo = fileInfo; + FileContent = fileContent; + } + + /// + /// Gets the file's mime type + /// + /// The file's mime type + public string? GetMimeType() + { + bool hasExtension = FileInfo.Filename.Contains("."); + if (!hasExtension) + { + return null; + } + + string fileExtension = FileInfo.Filename.Split(new char[] { '.' }).Last(); + if (MimeTypes.TryGetValue(fileExtension, out string mimeType)) + { + return mimeType; + } + else + { + return null; } } } diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/ChatMessage.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/ChatMessage.cs index 5de857ea4..b67b10f42 100644 --- a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/ChatMessage.cs +++ b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/ChatMessage.cs @@ -1,5 +1,6 @@ using Azure.AI.OpenAI; using Azure.AI.OpenAI.Chat; +using Microsoft.Bot.Schema; using Microsoft.Teams.AI.Exceptions; using Microsoft.Teams.AI.Utilities; using OpenAI.Chat; @@ -49,6 +50,10 @@ public class ChatMessage /// public IList? ToolCalls { get; set; } + /// + /// Attachments for the bot to send back. + /// + public List? Attachments { get; set; } /// /// Gets the content with the given type. diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/AssistantsPlanner.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/AssistantsPlanner.cs index 4e46d128d..442d1e25a 100644 --- a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/AssistantsPlanner.cs +++ b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/AssistantsPlanner.cs @@ -58,7 +58,7 @@ public AssistantsPlanner(AssistantsPlannerOptions options, ILoggerFactory? logge { Verify.ParamNotNull(options.Endpoint, "AssistantsPlannerOptions.Endpoint"); _client = _CreateClient(options.TokenCredential, options.Endpoint!); - _fileClient = _CreateFilesClient(options.TokenCredential, options.Endpoint!); + _fileClient = _CreateFileClient(options.TokenCredential, options.Endpoint!); } else if (options.ApiKey != null) { @@ -233,7 +233,7 @@ private async Task _GeneratePlanFromMessagesAsync(string threadId, string { foreach (MessageContent content in message.Content) { - ChatMessage chatMessage = new AssistantsMessage(content); + ChatMessage chatMessage = await _CreateAssistantMessageAsync(content); plan.Commands.Add(new PredictedSayCommand(chatMessage)); } } @@ -397,7 +397,7 @@ internal FileClient _CreateFileClient(string apiKey, string? endpoint = null) } } - internal FileClient _CreateFilesClient(TokenCredential tokenCredential, string endpoint) + internal FileClient _CreateFileClient(TokenCredential tokenCredential, string endpoint) { Verify.ParamNotNull(tokenCredential); Verify.ParamNotNull(endpoint); @@ -441,6 +441,64 @@ private async Task _CreateUserThreadMessageAsync(string threadId, return await _client.CreateMessageAsync(threadId, MessageRole.User, messages, options, cancellationToken); } + + private async Task _CreateAssistantMessageAsync(MessageContent messageContent, CancellationToken cancellationToken = default) + { + if (messageContent == null) + { + throw new ArgumentNullException(nameof(messageContent)); + } + + string textContent = messageContent.Text ?? ""; + MessageContext context = new(); + + List>> fileContentDownloadTasks = new(); + List>> fileInfoDownloadTasks = new(); + + for (int i = 0; i < messageContent.TextAnnotations.Count; i++) + { + TextAnnotation annotation = messageContent.TextAnnotations[i]; + if (annotation?.TextToReplace != null) + { + textContent = textContent.Replace(annotation.TextToReplace, $"[{i + 1}]"); + } + + if (annotation?.InputFileId != null) + { + // Retrieve file info object + // Neither `content` or `title` is provided in the annotations. + context.Citations.Add(new("Content not available", $"File {i + 1}", annotation.InputFileId)); + } + + if (annotation?.OutputFileId != null) + { + // Files generated by code interpretor tool. + fileContentDownloadTasks.Add(_fileClient.DownloadFileAsync(annotation.OutputFileId)); + fileInfoDownloadTasks.Add(_fileClient.GetFileAsync(annotation.OutputFileId)); + } + } + + List attachedFiles = new(); + if (fileContentDownloadTasks.Count > 0) + { + // Create attachments out of these downloaded files + // Wait for tasks to complete + ClientResult[] downloadedFileContent = await Task.WhenAll(fileContentDownloadTasks); + ClientResult[] downloadedFileInfo = await Task.WhenAll(fileInfoDownloadTasks); + + for (int i = 0; i < downloadedFileContent.Length; i++) + { + attachedFiles.Add(new OpenAIFile(downloadedFileInfo[i], downloadedFileContent[i])); + } + } + + AssistantsMessage message = new(messageContent, attachedFiles); + + message.Content = textContent; + message.Context = context; + + return message; + } } } #pragma warning restore OPENAI001