From 2f0ecac852dba6c1842f801493ff54e136cde1a8 Mon Sep 17 00:00:00 2001 From: SergeyMenshykh <68852919+SergeyMenshykh@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:03:19 +0000 Subject: [PATCH] .Net: Improve auto-recovery for Azure{OpenAI} models (#10275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation and Context Every time an AI model calls a function, and that call fails for whatever reason - hallucinated name, invalid arguments, etc. SK sends the failure details back to the model to allow it to correct (auto-recovery) itself and call the function one more time with the valid name, arguments, etc. Having tested this auto-recovery mechanism with a few modern {Azure}OpenAI models, it appeared that it does not work for the `Gpt-4(0314, 0613)` and `Gpt-4o(2024-08-06)` models. To make it work for the `Gpt-4o(2024-08-06)` model, the default error message needs to be extended either with the `Correct yourself` instruction, function name, or both. This solution is not enough to make the `Gpt-4(0314, 0613)` model's auto-recovery work, and as shown in the table below, the only way found so far to make it work is to use prompt engineering: | Error message/Model | Gpt-4(0314, 0613) | Gpt-4-Turbo(0125-Preview, 1106-Preview) | Gpt-4o mini(2024-07-18) | Gpt-4o(2024-08-06) | |----------------------------------------|--------------------|-------------------------------------------|---------------------------|---------------------| | `Error: Function call request for a function that wasn't defined` default message. | X | ✔️ | ✔️ | X | | Add `Correct yourself` instruction to the default error message. | X | ✔️ | ✔️ | ✔️ | | Add function name to the default error message. | X | ✔️ | ✔️ | ✔️ | | Add function name & `Correct yourself` instruction to the default error message.| X | ✔️ | ✔️ | ✔️ | | Use `You can call tools. If a tool call failed, correct yourself.` system message. | ✔️ | N/A | N/A | X | The `N/A` in the last row means that there is no need for prompt engineering because the AI model can auto-recover without it. ### Description This PR adds the `Correct yourself` option to the default error message to enable auto-recovery for the `Gpt-4o(2024-08-06)` model. Additional context: [Function Calling Reliability](https://github.com/microsoft/semantic-kernel/blob/main/docs/decisions/0063-function-calling-reliability.md) Closes: https://github.com/microsoft/semantic-kernel/issues/8472 --- .../connectors/AI/FunctionCalling/FunctionCallsProcessor.cs | 6 +++--- .../Utilities/AIConnectors/FunctionCallsProcessorTests.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dotnet/src/InternalUtilities/connectors/AI/FunctionCalling/FunctionCallsProcessor.cs b/dotnet/src/InternalUtilities/connectors/AI/FunctionCalling/FunctionCallsProcessor.cs index ef0db1510e88..5fe03ebb925d 100644 --- a/dotnet/src/InternalUtilities/connectors/AI/FunctionCalling/FunctionCallsProcessor.cs +++ b/dotnet/src/InternalUtilities/connectors/AI/FunctionCalling/FunctionCallsProcessor.cs @@ -312,14 +312,14 @@ private static bool TryValidateFunctionCall( // Check if the function call has an exception. if (functionCall.Exception is not null) { - errorMessage = $"Error: Function call processing failed. {functionCall.Exception.Message}"; + errorMessage = $"Error: Function call processing failed. Correct yourself. {functionCall.Exception.Message}"; return false; } // Make sure the requested function is one of the functions that was advertised to the AI model. if (!checkIfFunctionAdvertised(functionCall)) { - errorMessage = "Error: Function call request for a function that wasn't defined."; + errorMessage = "Error: Function call request for a function that wasn't defined. Correct yourself."; return false; } @@ -330,7 +330,7 @@ private static bool TryValidateFunctionCall( return true; } - errorMessage = "Error: Requested function could not be found."; + errorMessage = "Error: Requested function could not be found. Correct yourself."; return false; } diff --git a/dotnet/src/SemanticKernel.UnitTests/Utilities/AIConnectors/FunctionCallsProcessorTests.cs b/dotnet/src/SemanticKernel.UnitTests/Utilities/AIConnectors/FunctionCallsProcessorTests.cs index a4111bc9b5c0..6f0b40f8e82d 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Utilities/AIConnectors/FunctionCallsProcessorTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Utilities/AIConnectors/FunctionCallsProcessorTests.cs @@ -164,7 +164,7 @@ await this._sut.ProcessFunctionCallsAsync( Assert.Equal("MyPlugin", functionResult.PluginName); Assert.Equal("Function1", functionResult.FunctionName); - Assert.Equal("Error: Function call processing failed. Deserialization failed.", functionResult.Result); + Assert.Equal("Error: Function call processing failed. Correct yourself. Deserialization failed.", functionResult.Result); } [Fact] @@ -225,7 +225,7 @@ await this._sut.ProcessFunctionCallsAsync( Assert.Equal("MyPlugin", functionResult.PluginName); Assert.Equal("Function1", functionResult.FunctionName); - Assert.Equal("Error: Function call request for a function that wasn't defined.", functionResult.Result); + Assert.Equal("Error: Function call request for a function that wasn't defined. Correct yourself.", functionResult.Result); } [Fact] @@ -253,7 +253,7 @@ await this._sut.ProcessFunctionCallsAsync( Assert.Equal("MyPlugin", functionResult.PluginName); Assert.Equal("Function1", functionResult.FunctionName); - Assert.Equal("Error: Requested function could not be found.", functionResult.Result); + Assert.Equal("Error: Requested function could not be found. Correct yourself.", functionResult.Result); } [Theory]