Qwen3Coder bfcl - 0.95 simple/multiple

atobiszei · atobiszei · commit f0fa10a286b0 · 2025-09-22T10:59:36.000+02:00
0.825 parallel multiple
diff --git a/src/llm/BUILD b/src/llm/BUILD
@@ -115,6 +115,7 @@ ovms_cc_library(
         "@com_github_tencent_rapidjson//:rapidjson",
         "//src:libovmslogging",
         "//src:libovmsstring_utils",
+        "openai_request", # FIXME remove
     ] + select({
         "//conditions:default": ["//third_party:genai", ":llm_engine"],
         "//:not_genai_bin" : [":llm_engine"],
diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp
@@ -739,10 +739,8 @@ void OpenAIChatCompletionsHandler::incrementProcessedTokens(size_t numTokens) {
 
 absl::Status OpenAIChatCompletionsHandler::parseRequest(std::optional<uint32_t> maxTokensLimit, uint32_t bestOfLimit, std::optional<uint32_t> maxModelLength, std::optional<std::string> allowedLocalMediaPath) {
     absl::Status status = parseCommonPart(maxTokensLimit, bestOfLimit, maxModelLength);
-
     if (status != absl::OkStatus())
         return status;
-
     if (endpoint == Endpoint::COMPLETIONS)
         status = parseCompletionsPart();
     else
@@ -764,7 +762,7 @@ ParsedOutput OpenAIChatCompletionsHandler::parseOutputIfNeeded(const std::vector
     if (endpoint != Endpoint::CHAT_COMPLETIONS || outputParser == nullptr) {
         parsedOutput.content = tokenizer.decode(generatedIds);
     } else {
-        parsedOutput = outputParser->parse(generatedIds, areToolsAvailable());
+        parsedOutput = outputParser->parse(generatedIds, areToolsAvailable(), this->request.toolNameSchemaMap);
     }
     return parsedOutput;
 }
@@ -1058,6 +1056,7 @@ std::string OpenAIChatCompletionsHandler::serializeStreamingChunk(const std::str
     choice.AddMember("logprobs", Value(), allocator);
     if (endpoint == Endpoint::CHAT_COMPLETIONS) {
         if (outputParser != nullptr) {
+            // FIXME need tool maps for streaming
             std::optional<Document> delta = outputParser->parseChunk(chunkResponse, areToolsAvailable(), finishReason);
             if (!delta.has_value()) {
                 return "";
diff --git a/src/llm/apis/openai_request.hpp b/src/llm/apis/openai_request.hpp
@@ -34,6 +34,7 @@ struct StreamOptions {
     bool includeUsage = false;
 };
 
+using ToolsSchemas_t = std::map<std::string, std::string>;
 // Class that maps OpenAI request content.
 struct OpenAIChatCompletionsRequest {
     ov::genai::ChatHistory chatHistory;
@@ -74,7 +75,7 @@ struct OpenAIChatCompletionsRequest {
     // Schema for response_format handling
     std::optional<std::string> responseSchema{std::nullopt};
     // Map that holds tool names and schemas for their arguments
-    std::map<std::string, std::string> toolNameSchemaMap;
+    ToolsSchemas_t toolNameSchemaMap;
     // Holds value for tool_choice field as described in https://platform.openai.com/docs/api-reference/chat/create#chat_create-tool_choice
     std::string toolChoice;
 
diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp
@@ -30,6 +30,7 @@
 #pragma warning(pop)
 
 #include "partial_json_builder.hpp"
+#include "../apis/openai_request.hpp"
 
 namespace ovms {
 struct ToolCall {
@@ -49,6 +50,17 @@ struct ParsedOutput {
     std::string reasoning;
 };
 
+enum class ParameterType_t {
+    STRING,
+    NUMBER,
+    BOOLEAN,
+    ARRAY,
+    OBJECT,
+    UNKNOWN
+};
+using ParametersTypeMap_t = std::unordered_map<std::string, ParameterType_t>;       // param name -> param type
+using ToolsParameterTypeMap_t = std::unordered_map<std::string, ParametersTypeMap_t>;  // tool name -> (param name -> param type)
+
 class BaseOutputParser {
 protected:
     ov::genai::Tokenizer tokenizer;
@@ -79,7 +91,7 @@ class BaseOutputParser {
     // Parse model output and extract relevant information to parsedOutput fields. Raw generated tokens are provided as an argument.
     // Additionally parsedOutput.content is already filled with decoded content when this method is called, enabling chain or parsing.
     // Parser is also responsible for removing extracted part from the parsedOutput.content if necessary.
-    virtual void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) = 0;
+    virtual void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t& toolNameSchemaMap) = 0;
 
     // Parse model output chunk in the streaming mode. If in result of processing the chunk we cannot produce meaningful response, we return std::nullopt.
     // Otherwise we return a JSON object containing the delta that conforms to OpenAI API.
diff --git a/src/llm/io_processing/hermes3/tool_parser.cpp b/src/llm/io_processing/hermes3/tool_parser.cpp
@@ -31,7 +31,7 @@
 
 namespace ovms {
 
-void Hermes3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
+void Hermes3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
     const std::string startTag = "<tool_call>";
     const std::string endTag = "</tool_call>";
     std::vector<std::string> tools;
diff --git a/src/llm/io_processing/hermes3/tool_parser.hpp b/src/llm/io_processing/hermes3/tool_parser.hpp
@@ -55,7 +55,7 @@ class Hermes3ToolParser : public BaseOutputParser {
     explicit Hermes3ToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         return parsingStartTag;
diff --git a/src/llm/io_processing/llama3/tool_parser.cpp b/src/llm/io_processing/llama3/tool_parser.cpp
@@ -31,7 +31,7 @@
 #include "../utils.hpp"
 
 namespace ovms {
-void Llama3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
+void Llama3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
     // TODO: check if we can rely on decoded <|python_tag|> token to be present in the content, so we can drop multiple detokenizations and copies
     // and just extract substrings from the content and modify content in-place
 
diff --git a/src/llm/io_processing/llama3/tool_parser.hpp b/src/llm/io_processing/llama3/tool_parser.hpp
@@ -59,7 +59,7 @@ class Llama3ToolParser : public BaseOutputParser {
     explicit Llama3ToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         return parsingStartTag;
diff --git a/src/llm/io_processing/mistral/tool_parser.cpp b/src/llm/io_processing/mistral/tool_parser.cpp
@@ -32,7 +32,7 @@
 
 namespace ovms {
 
-void MistralToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
+void MistralToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
     std::vector<std::string> tools;
 
     if (parsedOutput.content.empty() || generatedTokens.size() <= 0) {
diff --git a/src/llm/io_processing/mistral/tool_parser.hpp b/src/llm/io_processing/mistral/tool_parser.hpp
@@ -39,7 +39,7 @@ class MistralToolParser : public BaseOutputParser {
     explicit MistralToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         static const std::string toolCallStartTag = "[TOOL_CALLS]";
diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp
@@ -188,7 +188,7 @@ std::string OutputParser::getToolParserStartTag() const {
     }
 }
 
-ParsedOutput OutputParser::parse(const std::vector<int64_t>& generatedTokens, const bool toolsAvailable) {
+ParsedOutput OutputParser::parse(const std::vector<int64_t>& generatedTokens, const bool toolsAvailable, const ToolsSchemas_t& toolNameSchemaMap) {
     // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure.
     // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens.
     // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output.
@@ -199,11 +199,12 @@ ParsedOutput OutputParser::parse(const std::vector<int64_t>& generatedTokens, co
     ParsedOutput parsedOutput;
     parsedOutput.content = tokenizer.decode(generatedTokens);
     if (reasoningParser) {
-        reasoningParser->parse(parsedOutput, generatedTokens);
+        reasoningParser->parse(parsedOutput, generatedTokens, toolNameSchemaMap);
     }
     // We run tool parser only if the parser is available and tools have been provided in the request.
     if (toolParser && toolsAvailable) {
-        toolParser->parse(parsedOutput, generatedTokens);
+        SPDLOG_ERROR("tool_parser schema size:{} address:{}", toolNameSchemaMap.size(), (void*)&toolNameSchemaMap);
+        toolParser->parse(parsedOutput, generatedTokens, toolNameSchemaMap);
     }
     return parsedOutput;
 }
diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp
@@ -22,6 +22,8 @@
 #include <unordered_set>
 
 #include "base_output_parser.hpp"
+// FIXME
+#include "../apis/openai_request.hpp"
 
 namespace ovms {
 
@@ -81,7 +83,7 @@ class OutputParser {
     std::string getToolParserStartTag() const;
 
     // Parse model output in the unary mode. Returns ParsedOutput containing data extracted by internal parsers.
-    ParsedOutput parse(const std::vector<int64_t>& generatedTokens, const bool toolsAvailable);
+    ParsedOutput parse(const std::vector<int64_t>& generatedTokens, const bool toolsAvailable, const ToolsSchemas_t& toolNameSchemaMap);
 
     // Parse model output chunk in the steaming mode. Returns a JSON object containing the delta that conforms to OpenAI API
     // or nullopt if no response can be produced.
diff --git a/src/llm/io_processing/phi4/tool_parser.cpp b/src/llm/io_processing/phi4/tool_parser.cpp
@@ -32,7 +32,7 @@
 
 namespace ovms {
 
-void Phi4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
+void Phi4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
     std::vector<std::string> tools;
 
     // Phi4 with vLLM template produces tool calls in the format:
diff --git a/src/llm/io_processing/phi4/tool_parser.hpp b/src/llm/io_processing/phi4/tool_parser.hpp
@@ -42,7 +42,7 @@ class Phi4ToolParser : public BaseOutputParser {
     explicit Phi4ToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         return parsingStartTag;
diff --git a/src/llm/io_processing/qwen3/reasoning_parser.cpp b/src/llm/io_processing/qwen3/reasoning_parser.cpp
@@ -30,7 +30,7 @@
 #include "../utils.hpp"
 
 namespace ovms {
-void Qwen3ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
+void Qwen3ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
     std::string startReasoningTag = getParsingStartTag();
     std::string endReasoningTag = getParsingEndTag();
     size_t startPos = parsedOutput.content.find(startReasoningTag);
diff --git a/src/llm/io_processing/qwen3/reasoning_parser.hpp b/src/llm/io_processing/qwen3/reasoning_parser.hpp
@@ -41,7 +41,7 @@ class Qwen3ReasoningParser : public BaseOutputParser {
     explicit Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         return parsingStartTag;
diff --git a/src/llm/io_processing/qwen3/tool_parser.cpp b/src/llm/io_processing/qwen3/tool_parser.cpp
@@ -54,7 +54,7 @@ static void trimNewline(std::string& str) {
         str.erase(str.begin());
     }
 }
-static bool isNumber(const std::string& s) {
+/*static bool isNumber(const std::string& s) {
     if (s.empty())
         return false;
     char* endptr = nullptr;
@@ -76,7 +76,7 @@ static bool isArrayLike(const std::string& s) {
     if (s.empty())
         return false;
     return (s.front() == '[' && s.back() == ']');
-}
+}*/
 /*static std::string toJsonLike(const std::string_view& s) {
     // use recursion to handle nested arrays and objects
     // first check if it is array
@@ -105,7 +105,7 @@ static bool isArrayLike(const std::string& s) {
 static std::string toJson(const std::vector<std::pair<std::string, std::string>>& items) {
     std::ostringstream oss;
     oss << "{";
-/*    for (size_t i = 0; i < items.size(); ++i) {
+    /*    for (size_t i = 0; i < items.size(); ++i) {
         const auto& [key, value] = items[i];
         oss << "\"" << key << "\": ";
         oss << toJsonLike(value);
@@ -119,14 +119,15 @@ static std::string toJson(const std::vector<std::pair<std::string, std::string>>
         const auto& [key, value] = items[i];
         oss << "\"" << key << "\": ";
 
-        if (isNumber(value) ||
+        oss << value;  // no quotes
+                       /*        if (isNumber(value) ||
             isBoolean(value) ||
             isJsonLike(value) ||
             isArrayLike(value)) {
             oss << value;  // no quotes
         } else {
             oss << "\"" << value << "\"";
-        }
+        }*/
 
         if (i + 1 < items.size()) {
             oss << ", ";
@@ -196,6 +197,76 @@ void Parser::removeToolCallsFromContent() {
         toolsEndStack.pop();
     }
 }
+
+// {"type":"object","properties":{"location":{"type":"string"},"provide_temperature":{"type":"boolean"}},"required":["location"]}
+// {"type":"object","required":["location"],"properties":{"location":{"type":"string","description":"The location for which to get the weather, in the format of 'City, State', such as 'San Francisco, CA' if State for the city exists. 'City, Country' if State for the city doesn't exist."},"unit":{"type":"string","description":"The unit of temperature for the weather report.","enum":["celsius","fahrenheit"],"default":"fahrenheit"}}}
+
+static const ParametersTypeMap_t parseToolSchema(const std::string& functionName, const std::string& schema) {
+    ParametersTypeMap_t result;
+    rapidjson::Document doc;
+    if (doc.Parse(schema.c_str()).HasParseError()) {
+        SPDLOG_WARN("Tool schema is not valid JSON for tool: {}, schema: {}", functionName, schema);
+        return result;
+    }
+    if (!doc.IsObject()) {
+        SPDLOG_WARN("Tool schema is not a JSON object for tool: {}, schema: {}", functionName, schema);
+        return result;
+    }
+    if (!doc.HasMember("properties") || !doc["properties"].IsObject()) {
+        SPDLOG_WARN("Tool schema does not have properties object for tool: {}, schema: {}", functionName, schema);
+        return result;
+    }
+    const rapidjson::Value& properties = doc["properties"];
+    for (auto it = properties.MemberBegin(); it != properties.MemberEnd(); ++it) {
+        if (!it->value.IsObject()) {
+            SPDLOG_WARN("Tool schema property: {} is not an object for tool: {}, schema: {}", it->name.GetString(), functionName, schema);
+            continue;
+        }
+        if (!it->value.HasMember("type") || !it->value["type"].IsString()) {
+            SPDLOG_WARN("Tool schema property: {} does not have type string for tool: {}, schema: {}", it->name.GetString(), functionName, schema);
+            continue;
+        }
+        std::string paramName = it->name.GetString();
+        std::string typeStr = it->value["type"].GetString();
+        ParameterType_t type = ParameterType_t::UNKNOWN;
+        if (typeStr == "string") {
+            type = ParameterType_t::STRING;
+        } else if (typeStr == "number" || typeStr == "integer") {
+            type = ParameterType_t::NUMBER;
+        } else if (typeStr == "boolean") {
+            type = ParameterType_t::BOOLEAN;
+        } else if (typeStr == "array") {
+            type = ParameterType_t::ARRAY;
+        } else if (typeStr == "object") {
+            type = ParameterType_t::OBJECT;
+        } else {
+            SPDLOG_WARN("Tool schema property: {} has unknown type: {} for tool: {}, schema: {}", paramName, typeStr, functionName, schema);
+        }
+        SPDLOG_TRACE("Tool:{} param:{} type:{}", functionName, paramName, typeStr);
+        result.emplace(paramName, type);
+    }
+    return result;
+}
+
+static std::string setCorrectValueType(std::string& inputValue, const std::string& currentParameterName, const ParametersTypeMap_t& parametersType) {
+    auto paramIt = parametersType.find(currentParameterName);
+    if (paramIt == parametersType.end()) {
+        SPDLOG_WARN("Parameter: {} schema not found for tool: {}, leaving as string", currentParameterName);
+        return inputValue;
+    }
+    if (paramIt->second == ParameterType_t::STRING) {
+        inputValue = "\"" + inputValue + "\"";
+        return inputValue;  // no change needed
+    }
+    if (paramIt->second == ParameterType_t::BOOLEAN) {
+        // in case of bool we need to convert to lower case
+        std::transform(inputValue.begin(), inputValue.end(), inputValue.begin(), ::tolower);
+        return inputValue;
+    }
+    // FIXME at error handling
+    return inputValue;
+}
+
 bool Parser::step(ToolCalls& toolCalls) {
     switch (currentState) {
     case Content: {
@@ -249,6 +320,13 @@ bool Parser::step(ToolCalls& toolCalls) {
         std::string parameterValue(content.substr(currentPosition, pos - currentPosition));
         if (this->removeNewlineAroundParameters)
             trimNewline(parameterValue);
+        // now we have parameter value in string format. We need to use toolsSchemas to determine if it is string, number, bool, array or object
+        auto paramIt = this->toolsParametersTypeMap.find(this->currentFunction.name);
+        if (paramIt == this->toolsParametersTypeMap.end()) {
+            SPDLOG_DEBUG("Tool schema not found for tool: {}, leaving parameter: {} as string", this->currentFunction.name, this->currentParameterName);
+        } else {
+            parameterValue = setCorrectValueType(parameterValue, this->currentParameterName, paramIt->second);
+        }
         currentFunction.parameters.emplace_back(this->currentParameterName, parameterValue);
         currentPosition = pos + Qwen3CoderToolParser::parameterEndTag.length();
         currentState = State::AfterParameter;
@@ -293,7 +371,7 @@ bool Parser::step(ToolCalls& toolCalls) {
     return true;
 }
 
-void Qwen3CoderToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
+void Qwen3CoderToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t& toolsSchemas) {
     // there may be multiple parameters per function, there may be multiple linses per parameter value
     // there is only one function per tool call
     // <tool_call>
@@ -303,11 +381,12 @@ void Qwen3CoderToolParser::parse(ParsedOutput& parsedOutput, const std::vector<i
     // </parameter>
     // </function>
     // </tool_call>a
-    //
-    // FIXME check for npos at each step
-    // For each if (itFunctionNameEnd == std::string::npos) {SPDLOG_ERROR("No tag end found"); return;}
-    // we need to replace it with macro
-    Parser parser(parsedOutput.content);
+    ToolsParameterTypeMap_t toolsParametersTypes;  // FIXME do it once per request
+    for (const auto& [toolName, schema] : toolsSchemas) {
+        toolsParametersTypes.emplace(toolName, parseToolSchema(toolName, schema));
+    }
+
+    Parser parser(parsedOutput.content, toolsParametersTypes);
     while (parser.step(parsedOutput.toolCalls))
         ;
     if (parser.currentState != Parser::State::End) {
@@ -320,5 +399,7 @@ void Qwen3CoderToolParser::parse(ParsedOutput& parsedOutput, const std::vector<i
 std::optional<rapidjson::Document> Qwen3CoderToolParser::parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) {
     return std::nullopt;  // FIXME
 }
-
+Parser::Parser(std::string& content, const ToolsParameterTypeMap_t& toolsParametersTypeMap) :
+    content(content),
+    toolsParametersTypeMap(toolsParametersTypeMap) {}
 }  // namespace ovms
diff --git a/src/llm/io_processing/qwen3/tool_parser.hpp b/src/llm/io_processing/qwen3/tool_parser.hpp
