Skip to content

Commit 18c6598

Browse files
Oceania2018Oceania2018
authored
Merge pull request #993 from gunpal5/master
feat: Added Google Multimodal Live APIs
2 parents 39e2746 + 382e9b1 commit 18c6598

26 files changed

+1951
-24
lines changed

‎BotSharp.sln

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Abstraction.Comput
141141
EndProject
142142
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse", "tests\BotSharp.Test.BrowserUse\BotSharp.Test.BrowserUse.csproj", "{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}"
143143
EndProject
144+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.LLM.Tests", "tests\BotSharp.LLM.Tests\BotSharp.LLM.Tests.csproj", "{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}"
145+
EndProject
144146
Global
145147
GlobalSection(SolutionConfigurationPlatforms) = preSolution
146148
Debug|Any CPU = Debug|Any CPU
@@ -589,6 +591,14 @@ Global
589591
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|Any CPU.Build.0 = Release|Any CPU
590592
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.ActiveCfg = Release|Any CPU
591593
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094}.Release|x64.Build.0 = Release|Any CPU
594+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
595+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|Any CPU.Build.0 = Debug|Any CPU
596+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|x64.ActiveCfg = Debug|Any CPU
597+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Debug|x64.Build.0 = Debug|Any CPU
598+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|Any CPU.ActiveCfg = Release|Any CPU
599+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|Any CPU.Build.0 = Release|Any CPU
600+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|x64.ActiveCfg = Release|Any CPU
601+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|x64.Build.0 = Release|Any CPU
592602
EndGlobalSection
593603
GlobalSection(SolutionProperties) = preSolution
594604
HideSolutionNode = FALSE
@@ -657,6 +667,7 @@ Global
657667
{B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
658668
{970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
659669
{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
670+
{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
660671
EndGlobalSection
661672
GlobalSection(ExtensibilityGlobals) = postSolution
662673
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

‎Directory.Packages.props

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
</PropertyGroup>
77
<ItemGroup>
88
<PackageVersion Include="EntityFramework" Version="6.4.4" />
9-
<PackageVersion Include="Google_GenerativeAI" Version="2.4.6" />
9+
<PackageVersion Include="Google_GenerativeAI" Version="2.5.5" />
10+
<PackageVersion Include="Google_GenerativeAI.Live" Version="2.5.5" />
1011
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
1112
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
1213
<PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
@@ -110,7 +111,8 @@
110111
<PackageVersion Include="MSTest.TestAdapter" Version="3.1.1" />
111112
<PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
112113
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
113-
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
114+
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
115+
<PackageVersion Include="Shouldly" Version="4.3.0" />
114116
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.5" />
115117
<PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.5" />
116118
</ItemGroup>
@@ -144,4 +146,4 @@
144146
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.25" />
145147
<PackageVersion Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="6.0.26" />
146148
</ItemGroup>
147-
</Project>
149+
</Project>

‎src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public class FunctionDef
2323
public string? Impact { get; set; }
2424

2525
[JsonPropertyName("parameters")]
26-
public FunctionParametersDef Parameters { get; set; } = new FunctionParametersDef();
26+
public FunctionParametersDef? Parameters { get; set; } = new FunctionParametersDef();
2727

2828
[JsonPropertyName("output")]
2929
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

‎src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -89,17 +89,20 @@ public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDial
8989
return responseMessage;
9090
}
9191

92-
public Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
92+
public Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations,
93+
Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
9394
{
9495
throw new NotImplementedException();
9596
}
9697

97-
public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
98+
public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations,
99+
Func<RoleDialogModel, Task> onMessageReceived)
98100
{
99101
throw new NotImplementedException();
100102
}
101103

102-
private (string, MessageParameters) PrepareOptions(Agent agent, List<RoleDialogModel> conversations, LlmModelSetting settings)
104+
private (string, MessageParameters) PrepareOptions(Agent agent, List<RoleDialogModel> conversations,
105+
LlmModelSetting settings)
103106
{
104107
var instruction = "";
105108
renderedInstructions = [];
@@ -178,8 +181,8 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
178181
var state = _services.GetRequiredService<IConversationStateService>();
179182
var temperature = decimal.Parse(state.GetState("temperature", "0.0"));
180183
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
181-
? tokens
182-
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
184+
? tokens
185+
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
183186

184187
var parameters = new MessageParameters()
185188
{
@@ -188,7 +191,7 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
188191
Model = settings.Name,
189192
Stream = false,
190193
Temperature = temperature,
191-
Tools = new List<Anthropic.SDK.Common.Tool>()
194+
Tools = new List<Anthropic.SDK.Common.Tool>()
192195
};
193196

194197
if (!string.IsNullOrEmpty(instruction))
@@ -197,9 +200,11 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
197200
{
198201
new SystemMessage(instruction)
199202
};
200-
};
203+
}
201204

202-
JsonSerializerOptions jsonSerializationOptions = new()
205+
;
206+
207+
JsonSerializerOptions? jsonSerializationOptions = new()
203208
{
204209
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
205210
Converters = { new JsonStringEnumConverter() },
@@ -238,7 +243,7 @@ public Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogM
238243

239244
private string GetPrompt(MessageParameters parameters)
240245
{
241-
var prompt = $"{string.Join("\r\n", parameters.System.Select(x => x.Text))}\r\n";
246+
var prompt = $"{string.Join("\r\n", (parameters.System ?? new List<SystemMessage>()).Select(x => x.Text))}\r\n";
242247
prompt += "\r\n[CONVERSATION]";
243248

244249
var verbose = string.Join("\r\n", parameters.Messages
@@ -272,17 +277,20 @@ private string GetPrompt(MessageParameters parameters)
272277
}));
273278
return $"{role}: {content}";
274279
}
280+
275281
return string.Empty;
276282
}));
277283

278284
prompt += $"\r\n{verbose}\r\n";
279285

280286
if (parameters.Tools != null && parameters.Tools.Count > 0)
281287
{
282-
var functions = string.Join("\r\n", parameters.Tools.Select(x =>
283-
{
284-
return $"\r\n{x.Function.Name}: {x.Function.Description}\r\n{JsonSerializer.Serialize(x.Function.Parameters)}";
285-
}));
288+
var functions = string.Join("\r\n",
289+
parameters.Tools.Select(x =>
290+
{
291+
return
292+
$"\r\n{x.Function.Name}: {x.Function.Description}\r\n{JsonSerializer.Serialize(x.Function.Parameters)}";
293+
}));
286294
prompt += $"\r\n[FUNCTIONS]\r\n{functions}\r\n";
287295
}
288296

@@ -293,4 +301,4 @@ public void SetModelName(string model)
293301
{
294302
_model = model;
295303
}
296-
}
304+
}

‎src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@
1212

1313
<ItemGroup>
1414
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
15+
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
1516
</ItemGroup>
1617

1718
<ItemGroup>
1819
<PackageReference Include="Google_GenerativeAI" />
20+
<PackageReference Include="Google_GenerativeAI.Live" />
1921
<PackageReference Include="LLMSharp.Google.Palm" />
2022
</ItemGroup>
2123

‎src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using BotSharp.Abstraction.Settings;
33
using BotSharp.Plugin.GoogleAi.Providers.Chat;
44
using BotSharp.Plugin.GoogleAI.Providers.Embedding;
5+
using BotSharp.Plugin.GoogleAi.Providers.Realtime;
56
using BotSharp.Plugin.GoogleAi.Providers.Text;
67

78
namespace BotSharp.Plugin.GoogleAi;
@@ -24,6 +25,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
2425
services.AddScoped<ITextCompletion, GeminiTextCompletionProvider>();
2526
services.AddScoped<IChatCompletion, PalmChatCompletionProvider>();
2627
services.AddScoped<IChatCompletion, GeminiChatCompletionProvider>();
28+
services.AddScoped<IRealTimeCompletion, GoogleRealTimeProvider>();
2729
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
2830
}
2931
}

‎src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,13 @@ public class GeminiChatCompletionProvider : IChatCompletion
2121
public string Provider => "google-ai";
2222
public string Model => _model;
2323

24+
private GoogleAiSettings _settings;
2425
public GeminiChatCompletionProvider(
2526
IServiceProvider services,
27+
GoogleAiSettings googleSettings,
2628
ILogger<GeminiChatCompletionProvider> logger)
2729
{
30+
_settings = googleSettings;
2831
_services = services;
2932
_logger = logger;
3033
}

‎src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,24 @@
11
using LLMSharp.Google.Palm;
2+
using Microsoft.Extensions.Logging;
23

34
namespace BotSharp.Plugin.GoogleAi.Providers;
45

56
public static class ProviderHelper
67
{
78
public static GenerativeAI.GoogleAi GetGeminiClient(string provider, string model, IServiceProvider services)
89
{
9-
var settingsService = services.GetRequiredService<ILlmProviderService>();
10-
var settings = settingsService.GetSetting(provider, model);
11-
var client = new GenerativeAI.GoogleAi(settings.ApiKey);
12-
return client;
10+
var aiSettings = services.GetRequiredService<GoogleAiSettings>();
11+
if (aiSettings == null || aiSettings.Gemini ==null || string.IsNullOrEmpty(aiSettings.Gemini.ApiKey))
12+
{
13+
var settingsService = services.GetRequiredService<ILlmProviderService>();
14+
var settings = settingsService.GetSetting(provider, model);
15+
var client = new GenerativeAI.GoogleAi(settings.ApiKey);
16+
return client;
17+
}
18+
else
19+
{
20+
return new GenerativeAI.GoogleAi(aiSettings.Gemini.ApiKey);
21+
}
1322
}
1423

1524
public static GooglePalmClient GetPalmClient(string provider, string model, IServiceProvider services)

0 commit comments

Comments
 (0)