Skip to content

Commit

Permalink
[C#] feat: Azure OpenAI On Your Data support (#1478)
Browse files Browse the repository at this point in the history
## Linked issues

closes: #1420  (issue number)

## Details
* Added addtional data properties to `config.json` 
* Added `08.datasource.azureopenai` sample


## Attestation Checklist

- [x] My code follows the style guidelines of this project

- I have checked for/fixed spelling, linting, and other errors
- I have commented my code for clarity
- I have made corresponding changes to the documentation (updating the
doc strings in the code is sufficient)
- My changes generate no new warnings
- I have added tests that validates my changes, and provides sufficient
test coverage. I have tested with:
  - Local testing
  - E2E testing in Teams
- New and existing unit tests pass locally with my changes

---------

Co-authored-by: Alex Acebo <[email protected]>
  • Loading branch information
singhk97 and aacebo authored Apr 11, 2024
1 parent 64c3547 commit 63fdd1e
Show file tree
Hide file tree
Showing 31 changed files with 994 additions and 20 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public void Test_Constructor_AzureOpenAI_InvalidAzureApiVersion()
var options = new AzureOpenAIModelOptions("test-key", "test-deployment", "https://test.openai.azure.com/");
var versions = new List<string>
{
"2022-12-01", "2023-05-15", "2023-06-01-preview", "2023-07-01-preview", "2023-08-01-preview", "2023-09-01-preview"
"2022-12-01", "2023-05-15", "2023-06-01-preview", "2023-07-01-preview", "2024-02-15-preview", "2024-03-01-preview"
};

// Act
Expand Down Expand Up @@ -177,7 +177,7 @@ public async void Test_CompletePromptAsync_AzureOpenAI_Text()
LogRequests = true
};
var clientMock = new Mock<OpenAIClient>();
var choice = CreateChoice("test-choice", 0, null, null, null);
var choice = CreateChoice("test-choice", 0, null, null, null, null);
var usage = CreateCompletionsUsage(0, 0, 0);
var completions = CreateCompletions("test-id", DateTimeOffset.UtcNow, new List<Choice> { choice }, usage);
Response response = new TestResponse(200, string.Empty);
Expand Down Expand Up @@ -308,8 +308,8 @@ public async void Test_CompletePromptAsync_AzureOpenAI_Chat()
LogRequests = true
};
var clientMock = new Mock<OpenAIClient>();
var chatResponseMessage = CreateChatResponseMessage(Azure.AI.OpenAI.ChatRole.Assistant, "test-choice", null, null, null);
var chatChoice = CreateChatChoice(chatResponseMessage, 0, null, null, null, null, null);
var chatResponseMessage = CreateChatResponseMessage(Azure.AI.OpenAI.ChatRole.Assistant, "test-choice", null, null, null, null);
var chatChoice = CreateChatChoice(chatResponseMessage, null, 0, null, null, null, null, null, null);
var usage = CreateCompletionsUsage(0, 0, 0);
var chatCompletions = CreateChatCompletions("test-id", DateTimeOffset.UtcNow, new List<ChatChoice> { chatChoice }, usage);
Response response = new TestResponse(200, string.Empty);
Expand All @@ -328,10 +328,10 @@ public async void Test_CompletePromptAsync_AzureOpenAI_Chat()
Assert.Equal("test-choice", result.Message.Content);
}

private static Choice CreateChoice(string text, int index, ContentFilterResultsForChoice? contentFilterResults, CompletionsLogProbabilityModel? logProbabilityModel, CompletionsFinishReason? finishReason)
private static Choice CreateChoice(string text, int index, ContentFilterResultsForChoice? contentFilterResults, CompletionsLogProbabilityModel? logProbabilityModel, CompletionsFinishReason? finishReason, IDictionary<string, BinaryData>? serializedAdditionalRawData)
{
Type[] paramTypes = new Type[] { typeof(string), typeof(int), typeof(ContentFilterResultsForChoice), typeof(CompletionsLogProbabilityModel), typeof(CompletionsFinishReason) };
object[] paramValues = new object[] { text, index, contentFilterResults, logProbabilityModel!, finishReason! };
Type[] paramTypes = new Type[] { typeof(string), typeof(int), typeof(ContentFilterResultsForChoice), typeof(CompletionsLogProbabilityModel), typeof(CompletionsFinishReason), typeof(IDictionary<string, BinaryData>) };
object[] paramValues = new object[] { text, index, contentFilterResults!, logProbabilityModel!, finishReason!, serializedAdditionalRawData! };
return Construct<Choice>(paramTypes, paramValues);
}

Expand All @@ -349,17 +349,17 @@ private static Completions CreateCompletions(string id, DateTimeOffset created,
return Construct<Completions>(paramTypes, paramValues);
}

private static ChatResponseMessage CreateChatResponseMessage(Azure.AI.OpenAI.ChatRole role, string content, IReadOnlyList<Azure.AI.OpenAI.ChatCompletionsToolCall>? toolCalls, Azure.AI.OpenAI.FunctionCall? functionCall, AzureChatExtensionsMessageContext? azureExtensionsContext)
private static ChatResponseMessage CreateChatResponseMessage(Azure.AI.OpenAI.ChatRole role, string content, IReadOnlyList<Azure.AI.OpenAI.ChatCompletionsToolCall>? toolCalls, Azure.AI.OpenAI.FunctionCall? functionCall, AzureChatExtensionsMessageContext? azureExtensionsContext, IDictionary<string, BinaryData>? serializedAdditionalRawData)
{
Type[] paramTypes = new Type[] { typeof(Azure.AI.OpenAI.ChatRole), typeof(string), typeof(IReadOnlyList<Azure.AI.OpenAI.ChatCompletionsToolCall>), typeof(Azure.AI.OpenAI.FunctionCall), typeof(AzureChatExtensionsMessageContext) };
object[] paramValues = new object[] { role, content, toolCalls!, functionCall!, azureExtensionsContext! };
Type[] paramTypes = new Type[] { typeof(Azure.AI.OpenAI.ChatRole), typeof(string), typeof(IReadOnlyList<Azure.AI.OpenAI.ChatCompletionsToolCall>), typeof(Azure.AI.OpenAI.FunctionCall), typeof(AzureChatExtensionsMessageContext), typeof(IDictionary<string, BinaryData>) };
object[] paramValues = new object[] { role, content, toolCalls!, functionCall!, azureExtensionsContext!, serializedAdditionalRawData! };
return Construct<ChatResponseMessage>(paramTypes, paramValues);
}

private static ChatChoice CreateChatChoice(ChatResponseMessage message, int index, CompletionsFinishReason? finishReason, ChatFinishDetails? finishDetails, ChatResponseMessage? internalStreamingDeltaMessage, ContentFilterResultsForChoice? contentFilterResults, AzureChatEnhancements? enhancements)
private static ChatChoice CreateChatChoice(ChatResponseMessage message, ChatChoiceLogProbabilityInfo? logProbabilityInfo, int index, CompletionsFinishReason? finishReason, ChatFinishDetails? finishDetails, ChatResponseMessage? internalStreamingDeltaMessage, ContentFilterResultsForChoice? contentFilterResults, AzureChatEnhancements? enhancements, IDictionary<string, BinaryData>? serializedAdditionalRawData)
{
Type[] paramTypes = new Type[] { typeof(ChatResponseMessage), typeof(int), typeof(CompletionsFinishReason), typeof(ChatFinishDetails), typeof(ChatResponseMessage), typeof(ContentFilterResultsForChoice), typeof(AzureChatEnhancements) };
object[] paramValues = new object[] { message, index, finishReason!, finishDetails!, internalStreamingDeltaMessage!, contentFilterResults!, enhancements! };
Type[] paramTypes = new Type[] { typeof(ChatResponseMessage), typeof(ChatChoiceLogProbabilityInfo), typeof(int), typeof(CompletionsFinishReason), typeof(ChatFinishDetails), typeof(ChatResponseMessage), typeof(ContentFilterResultsForChoice), typeof(AzureChatEnhancements), typeof(IDictionary<string, BinaryData>) };
object[] paramValues = new object[] { message, logProbabilityInfo!, index, finishReason!, finishDetails!, internalStreamingDeltaMessage!, contentFilterResults!, enhancements!, serializedAdditionalRawData! };
return Construct<ChatChoice>(paramTypes, paramValues);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.12" />
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.15" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
<PackageReference Include="Microsoft.Bot.Builder" Version="4.22.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,8 @@ public async Task<EmbeddingsResponse> CreateEmbeddingsAsync(IList<string> inputs
case "2023-05-15": return ServiceVersion.V2023_05_15;
case "2023-06-01-preview": return ServiceVersion.V2023_06_01_Preview;
case "2023-07-01-preview": return ServiceVersion.V2023_07_01_Preview;
case "2023-08-01-preview": return ServiceVersion.V2023_08_01_Preview;
case "2023-09-01-preview": return ServiceVersion.V2023_09_01_Preview;
case "2024-02-15-preview": return ServiceVersion.V2024_02_15_Preview;
case "2024-03-01-preview": return ServiceVersion.V2024_03_01_Preview;
default:
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ public static ChatMessage ToChatMessage(this ChatResponseMessage chatMessage)

}

message.Context = new MessageContext();
if (chatMessage.AzureExtensionsContext?.Intent != null)
{
message.Context.Intent = chatMessage.AzureExtensionsContext.Intent;
}

IReadOnlyList<AzureChatExtensionDataSourceResponseCitation>? citations = chatMessage.AzureExtensionsContext?.Citations;
if (citations != null)
{
foreach (AzureChatExtensionDataSourceResponseCitation citation in citations)
{
message.Context.Citations.Add(new Citation(citation.Content, citation.Title, citation.Url));
};
}

return message;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ public class ChatMessage
/// </summary>
public string? ToolCallId { get; set; }

/// <summary>
/// The context used for this message.
/// </summary>
public MessageContext? Context { get; set; }

/// <summary>
/// The tool calls generated by the model, such as function calls.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
using Microsoft.Teams.AI.Utilities;

namespace Microsoft.Teams.AI.AI.Models
{
/// <summary>
/// The message context.
/// </summary>
public class MessageContext
{
/// <summary>
/// Citations used in the message.
/// </summary>
public IList<Citation> Citations { get; } = new List<Citation>();

/// <summary>
/// The intent of the message.
/// </summary>
public string Intent { get; set; } = string.Empty;
}

/// <summary>
/// Citations used in the message.
/// </summary>
public class Citation
{
/// <summary>
/// The content of the citation.
/// </summary>
public string Content { get; set; }

/// <summary>
/// The title of the citation.
/// </summary>
public string Title { get; set; }

/// <summary>
/// The URL of the citation.
/// </summary>
public string Url { get; set; }

/// <summary>
/// Constructs a citation.
/// </summary>
/// <param name="content">The content of the citation.</param>
/// <param name="title">The title of the citation.</param>
/// <param name="url">The url of the citation.</param>
public Citation(string content, string title, string url)
{
Verify.ParamNotNull(content);
Verify.ParamNotNull(title);
Verify.ParamNotNull(url);

Content = content;
Title = title;
Url = url;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using Azure.Core;
using Azure.Core.Pipeline;
using Microsoft.Bot.Builder;
using Microsoft.Extensions.Azure;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Teams.AI.AI.Prompts;
Expand All @@ -11,6 +12,7 @@
using Microsoft.Teams.AI.Exceptions;
using Microsoft.Teams.AI.State;
using Microsoft.Teams.AI.Utilities;
using System.ClientModel.Primitives;
using System.Net;
using System.Text.Json;
using static Azure.AI.OpenAI.OpenAIClientOptions;
Expand Down Expand Up @@ -93,7 +95,7 @@ public OpenAIModel(AzureOpenAIModelOptions options, ILoggerFactory? loggerFactor
{
throw new ArgumentException($"Model created with an invalid endpoint of `{options.AzureEndpoint}`. The endpoint must be a valid HTTPS url.");
}
string apiVersion = options.AzureApiVersion ?? "2023-05-15";
string apiVersion = options.AzureApiVersion ?? "2024-02-15-preview";
ServiceVersion? serviceVersion = ConvertStringToServiceVersion(apiVersion);
if (serviceVersion == null)
{
Expand Down Expand Up @@ -240,6 +242,9 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
FrequencyPenalty = (float)promptTemplate.Configuration.Completion.FrequencyPenalty,
};

IDictionary<string, JsonElement>? additionalData = promptTemplate.Configuration.Completion.AdditionalData;
AddAzureChatExtensionConfigurations(chatCompletionsOptions, additionalData);

Response? rawResponse;
Response<ChatCompletions>? chatCompletionsResponse = null;
PromptResponse promptResponse = new();
Expand Down Expand Up @@ -295,11 +300,42 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
case "2023-05-15": return ServiceVersion.V2023_05_15;
case "2023-06-01-preview": return ServiceVersion.V2023_06_01_Preview;
case "2023-07-01-preview": return ServiceVersion.V2023_07_01_Preview;
case "2023-08-01-preview": return ServiceVersion.V2023_08_01_Preview;
case "2023-09-01-preview": return ServiceVersion.V2023_09_01_Preview;
case "2024-02-15-preview": return ServiceVersion.V2024_02_15_Preview;
case "2024-03-01-preview": return ServiceVersion.V2024_03_01_Preview;
default:
return null;
}
}

private void AddAzureChatExtensionConfigurations(ChatCompletionsOptions options, IDictionary<string, JsonElement>? additionalData)
{
if (additionalData == null)
{
return;
}

if (additionalData != null && additionalData.TryGetValue("data_sources", out JsonElement array))
{
List<AzureChatExtensionConfiguration> configurations = new();
List<object> entries = array.Deserialize<List<object>>()!;
foreach (object item in entries)
{
AzureChatExtensionConfiguration? dataSourceItem = ModelReaderWriter.Read<AzureChatExtensionConfiguration>(BinaryData.FromObjectAsJson(item));
if (dataSourceItem != null)
{
configurations.Add(dataSourceItem);
}
}

if (configurations.Count > 0)
{
options.AzureExtensionsOptions = new();
foreach (AzureChatExtensionConfiguration configuration in configurations)
{
options.AzureExtensionsOptions.Extensions.Add(configuration);
}
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,12 @@ public class CompletionConfiguration
[JsonPropertyOrder(11)]
public double TopP { get; set; } = 0.0f;

/// <summary>
/// Additional data provided in the completion configuration.
/// </summary>
[JsonExtensionData]
public IDictionary<string, JsonElement>? AdditionalData { get; set; } = null;

/// <summary>
/// Completion Type
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -921,6 +921,12 @@ private async Task _OnTurnAsync(ITurnContext turnContext, CancellationToken canc
}
}

// Populate {{$temp.input}}
if ((turnState.Temp.Input == null || turnState.Temp.Input.Length == 0) && turnContext.Activity.Text != null)
{
turnState.Temp.Input = turnContext.Activity.Text;
}

bool eventHandlerCalled = false;

// Run any RouteSelectors in this._invokeRoutes first if the incoming Teams activity.type is "Invoke".
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
<ItemGroup>
<PackageReference Include="AdaptiveCards" Version="3.1.0" />
<PackageReference Include="Azure.AI.ContentSafety" Version="1.0.0-beta.1" />
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.12" />
<PackageReference Include="Azure.AI.OpenAI" Version="1.0.0-beta.15" />
<PackageReference Include="JsonSchema.Net" Version="5.5.1" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
<PackageReference Include="Microsoft.Bot.Builder" Version="4.22.2" />
Expand Down
12 changes: 12 additions & 0 deletions dotnet/samples/08.datasource.azureopenai/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
AzureOpenAIBot/bin/Debug
AzureOpenAIBot/obj
AzureOpenAIBot.csproj.user
AzureOpenAIBot.user
appsettings.Development.json
appsettings.json
appsettings.TestTool.json
.vs/
**/env/
bin/Debug/
obj/
appPackage/
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
using Microsoft.Bot.Builder.TraceExtensions;
using Microsoft.Teams.AI;

namespace AzureOpenAIBot
{
public class AdapterWithErrorHandler : TeamsAdapter
{
public AdapterWithErrorHandler(IConfiguration configuration, ILogger<TeamsAdapter> logger)
: base(configuration, null, logger)
{
OnTurnError = async (turnContext, exception) =>
{
// Log any leaked exception from the application.
// NOTE: In production environment, you should consider logging this to
// Azure Application Insights. Visit https://aka.ms/bottelemetry to see how
// to add telemetry capture to your bot.
logger.LogError(exception, $"[OnTurnError] unhandled error : {exception.Message}");
// Send a message to the user
await turnContext.SendActivityAsync($"The bot encountered an unhandled error: {exception.Message}");
await turnContext.SendActivityAsync("To continue to run this bot, please fix the bot source code.");
// Send a trace activity
await turnContext.TraceActivityAsync("OnTurnError Trace", exception.Message, "https://www.botframework.com/schemas/error", "TurnError");
};
}
}
}
38 changes: 38 additions & 0 deletions dotnet/samples/08.datasource.azureopenai/AzureOpenAIBot.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>

<ItemGroup>
<ProjectCapability Include="TeamsFx" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Bot.Builder" Version="4.21.1" />
<PackageReference Include="Microsoft.Bot.Builder.Integration.AspNet.Core" Version="4.21.1" />
<PackageReference Include="Microsoft.Bot.Connector" Version="4.21.1" />
<PackageReference Include="Microsoft.Teams.AI" Version="1.1.*" />
</ItemGroup>

<!-- Exclude Teams Toolkit files from build output, but can still be viewed from Solution Explorer -->
<ItemGroup>
<Content Remove="appPackage/**/*" />
<None Include="appPackage/**/*" />
<None Include="env/**/*" />
<Content Remove="infra/**/*" />
<None Include="infra/**/*" />
</ItemGroup>

<!-- Exclude local settings from publish -->
<ItemGroup>
<Content Remove="appsettings.Development.json" />
<Content Include="appsettings.Development.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<CopyToPublishDirectory>None</CopyToPublishDirectory>
</Content>
</ItemGroup>

</Project>
Loading

0 comments on commit 63fdd1e

Please sign in to comment.