feat: Added Google Multimodal Live APIs

BotSharp.sln

@@ -141,6 +141,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Abstraction.Comput
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.LLM.Tests", "tests\BotSharp.LLM.Tests\BotSharp.LLM.Tests.csproj", "{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -589,6 +591,14 @@ Global
 		{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.Build.0 = Release|Any CPU
 		{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.ActiveCfg = Release|Any CPU
 		{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.Build.0 = Release|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|x64.Build.0 = Debug|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|x64.ActiveCfg = Release|Any CPU
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -657,6 +667,7 @@ Global
 		{B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
 		{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
+		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

Directory.Packages.props

@@ -6,7 +6,8 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="EntityFramework" Version="6.4.4" />
-    <PackageVersion Include="Google_GenerativeAI" Version="2.4.6" />
+    <PackageVersion Include="Google_GenerativeAI" Version="2.5.5" />
+    <PackageVersion Include="Google_GenerativeAI.Live" Version="2.5.5" />
     <PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
     <PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
@@ -110,7 +111,8 @@
     <PackageVersion Include="MSTest.TestAdapter" Version="3.1.1" />
     <PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
-    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />  
+    <PackageVersion Include="Shouldly" Version="4.3.0" />
     <PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.5" /> 
     <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.5" />
   </ItemGroup>
@@ -144,4 +146,4 @@
     <PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" />
     <PackageVersion Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="6.0.26" />
   </ItemGroup>
-</Project>
+</Project>

src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs

@@ -23,7 +23,7 @@ public class FunctionDef
     public string? Impact { get; set; }
 
     [JsonPropertyName("parameters")]
-    public FunctionParametersDef Parameters { get; set; } = new FunctionParametersDef();
+    public FunctionParametersDef? Parameters { get; set; } = new FunctionParametersDef();
 
     [JsonPropertyName("output")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs

@@ -89,17 +89,20 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
         return responseMessage;
     }
 
-    public Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
+    public Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations,
+        Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
     {
         throw new NotImplementedException();
     }
 
-    public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
+    public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations,
+        Func<RoleDialogModel, Task> onMessageReceived)
     {
         throw new NotImplementedException();
     }
 
-    private (string, MessageParameters) PrepareOptions(Agent agent, List<RoleDialogModel> conversations, LlmModelSetting settings)
+    private (string, MessageParameters) PrepareOptions(Agent agent, List<RoleDialogModel> conversations,
+        LlmModelSetting settings)
     {
         var instruction = "";
         renderedInstructions = [];
@@ -178,8 +181,8 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
         var state = _services.GetRequiredService<IConversationStateService>();
         var temperature = decimal.Parse(state.GetState("temperature", "0.0"));
         var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
-                            ? tokens
-                            : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
+            ? tokens
+            : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
 
         var parameters = new MessageParameters()
         {
@@ -188,7 +191,7 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
             Model = settings.Name,
             Stream = false,
             Temperature = temperature,
-            Tools = new List<Anthropic.SDK.Common.Tool>() 
+            Tools = new List<Anthropic.SDK.Common.Tool>()
         };
 
         if (!string.IsNullOrEmpty(instruction))
@@ -197,9 +200,11 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
             {
                 new SystemMessage(instruction)
             };
-        };
+        }
 
-        JsonSerializerOptions jsonSerializationOptions = new()
+        ;
+
+        JsonSerializerOptions? jsonSerializationOptions = new()
         {
             DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
             Converters = { new JsonStringEnumConverter() },
@@ -238,7 +243,7 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
 
     private string GetPrompt(MessageParameters parameters)
     {
-        var prompt = $"{string.Join("\r\n", parameters.System.Select(x => x.Text))}\r\n";
+        var prompt = $"{string.Join("\r\n", (parameters.System ?? new List<SystemMessage>()).Select(x => x.Text))}\r\n";
         prompt += "\r\n[CONVERSATION]";
 
         var verbose = string.Join("\r\n", parameters.Messages
@@ -272,17 +277,20 @@ private string GetPrompt(MessageParameters parameters)
                     }));
                     return $"{role}: {content}";
                 }
+
                 return string.Empty;
             }));
 
         prompt += $"\r\n{verbose}\r\n";
 
         if (parameters.Tools != null && parameters.Tools.Count > 0)
         {
-            var functions = string.Join("\r\n", parameters.Tools.Select(x =>
-            {
-                return $"\r\n{x.Function.Name}: {x.Function.Description}\r\n{JsonSerializer.Serialize(x.Function.Parameters)}";
-            }));
+            var functions = string.Join("\r\n",
+                parameters.Tools.Select(x =>
+                {
+                    return
+                        $"\r\n{x.Function.Name}: {x.Function.Description}\r\n{JsonSerializer.Serialize(x.Function.Parameters)}";
+                }));
             prompt += $"\r\n[FUNCTIONS]\r\n{functions}\r\n";
         }
 
@@ -293,4 +301,4 @@ public void SetModelName(string model)
     {
         _model = model;
     }
-}
+}

src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj

@@ -12,10 +12,12 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="Google_GenerativeAI" />
+    <PackageReference Include="Google_GenerativeAI.Live" />
     <PackageReference Include="LLMSharp.Google.Palm" />
   </ItemGroup>
 

src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs

@@ -2,6 +2,7 @@
 using BotSharp.Abstraction.Settings;
 using BotSharp.Plugin.GoogleAi.Providers.Chat;
 using BotSharp.Plugin.GoogleAI.Providers.Embedding;
+using BotSharp.Plugin.GoogleAi.Providers.Realtime;
 using BotSharp.Plugin.GoogleAi.Providers.Text;
 
 namespace BotSharp.Plugin.GoogleAi;
@@ -24,6 +25,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<ITextCompletion, GeminiTextCompletionProvider>();
         services.AddScoped<IChatCompletion, PalmChatCompletionProvider>();
         services.AddScoped<IChatCompletion, GeminiChatCompletionProvider>();
+        services.AddScoped<IRealTimeCompletion, GoogleRealTimeProvider>();
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
     }
 }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs

@@ -21,10 +21,13 @@ public class GeminiChatCompletionProvider : IChatCompletion
     public string Provider => "google-ai";
     public string Model => _model;
 
+    private GoogleAiSettings _settings;
     public GeminiChatCompletionProvider(
         IServiceProvider services,
+        GoogleAiSettings googleSettings,
         ILogger<GeminiChatCompletionProvider> logger)
     {
+        _settings = googleSettings;
         _services = services;
         _logger = logger;
     }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs

@@ -1,15 +1,24 @@
 using LLMSharp.Google.Palm;
+using Microsoft.Extensions.Logging;
 
 namespace BotSharp.Plugin.GoogleAi.Providers;
 
 public static class ProviderHelper
 {
     public static GenerativeAI.GoogleAi GetGeminiClient(string provider, string model, IServiceProvider services)
     {
-        var settingsService = services.GetRequiredService<ILlmProviderService>();
-        var settings = settingsService.GetSetting(provider, model);
-        var client = new GenerativeAI.GoogleAi(settings.ApiKey);
-        return client;
+        var aiSettings = services.GetRequiredService<GoogleAiSettings>();
+        if (aiSettings == null || aiSettings.Gemini ==null || string.IsNullOrEmpty(aiSettings.Gemini.ApiKey))
+        {
+            var settingsService = services.GetRequiredService<ILlmProviderService>();
+            var settings = settingsService.GetSetting(provider, model);
+            var client = new GenerativeAI.GoogleAi(settings.ApiKey);
+            return client;
+        }
+        else
+        {
+            return new GenerativeAI.GoogleAi(aiSettings.Gemini.ApiKey);
+        }
     }
 
     public static GooglePalmClient GetPalmClient(string provider, string model, IServiceProvider services)