Streaming working

Accuracy on BFCL simple multiple as in unary

Author: atobiszei

src/llm/apis/openai_completions.cpp

@@ -760,9 +760,9 @@ ParsedOutput OpenAIChatCompletionsHandler::parseOutputIfNeeded(const std::vector
     OVMS_PROFILE_FUNCTION();
     ParsedOutput parsedOutput;
     if (endpoint != Endpoint::CHAT_COMPLETIONS || outputParser == nullptr) {
-        parsedOutput.content = tokenizer.decode(generatedIds);
+        parsedOutput.content = this->tokenizer.decode(generatedIds);
     } else {
-        parsedOutput = outputParser->parse(generatedIds, areToolsAvailable(), this->request.toolNameSchemaMap);
+        parsedOutput = outputParser->parse(generatedIds, this->areToolsAvailable());
     }
     return parsedOutput;
 }

src/llm/apis/openai_completions.hpp

@@ -86,8 +86,9 @@ class OpenAIChatCompletionsHandler {
         endpoint(endpoint),
         created(creationTime),
         tokenizer(tokenizer) {
+        // FIXME we should delay creating output parser until we have requiest with toolNameSchemaMap parsed
         if (!toolParserName.empty() || !reasoningParserName.empty()) {
-            outputParser = std::make_unique<OutputParser>(tokenizer, toolParserName, reasoningParserName);
+            outputParser = std::make_unique<OutputParser>(tokenizer, toolParserName, reasoningParserName, this->request.toolNameSchemaMap);
         }
     }
 

src/llm/io_processing/base_output_parser.hpp

@@ -93,7 +93,7 @@ class BaseOutputParser {
     // Parse model output and extract relevant information to parsedOutput fields. Raw generated tokens are provided as an argument.
     // Additionally parsedOutput.content is already filled with decoded content when this method is called, enabling chain or parsing.
     // Parser is also responsible for removing extracted part from the parsedOutput.content if necessary.
-    virtual void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t& toolNameSchemaMap) = 0;
+    virtual void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) = 0;
 
     // Parse model output chunk in the streaming mode. If in result of processing the chunk we cannot produce meaningful response, we return std::nullopt.
     // Otherwise we return a JSON object containing the delta that conforms to OpenAI API.

src/llm/io_processing/hermes3/tool_parser.cpp

@@ -31,7 +31,7 @@
 
 namespace ovms {
 
-void Hermes3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
+void Hermes3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
     const std::string startTag = "<tool_call>";
     const std::string endTag = "</tool_call>";
     std::vector<std::string> tools;

src/llm/io_processing/hermes3/tool_parser.hpp

@@ -55,7 +55,7 @@ class Hermes3ToolParser : public BaseOutputParser {
     explicit Hermes3ToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         return parsingStartTag;

src/llm/io_processing/llama3/tool_parser.cpp

@@ -31,7 +31,7 @@
 #include "../utils.hpp"
 
 namespace ovms {
-void Llama3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
+void Llama3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
     // TODO: check if we can rely on decoded <|python_tag|> token to be present in the content, so we can drop multiple detokenizations and copies
     // and just extract substrings from the content and modify content in-place
 

src/llm/io_processing/llama3/tool_parser.hpp

@@ -38,7 +38,7 @@ class Llama3ToolParser : public BaseOutputParser {
     const std::string parsingEndTag = "";
 
     // Id of the <|python_tag|> which is a special token used to indicate the start of a tool calls
-    static const int64_t botTokenId = 128010;
+    int64_t botTokenId = 128010;
     // ";" is used as a separator between tool calls in the response
     std::string separator = ";";
 
@@ -59,7 +59,7 @@ class Llama3ToolParser : public BaseOutputParser {
     explicit Llama3ToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         return parsingStartTag;

src/llm/io_processing/mistral/tool_parser.cpp

@@ -32,7 +32,7 @@
 
 namespace ovms {
 
-void MistralToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) {
+void MistralToolParser::parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) {
     std::vector<std::string> tools;
 
     if (parsedOutput.content.empty() || generatedTokens.size() <= 0) {

src/llm/io_processing/mistral/tool_parser.hpp

@@ -39,7 +39,7 @@ class MistralToolParser : public BaseOutputParser {
     explicit MistralToolParser(ov::genai::Tokenizer& tokenizer) :
         BaseOutputParser(tokenizer) {}
 
-    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens, const ToolsSchemas_t&) override;
+    void parse(ParsedOutput& parsedOutput, const std::vector<int64_t>& generatedTokens) override;
     std::optional<rapidjson::Document> parseChunk(const std::string& chunk, ov::genai::GenerationFinishReason finishReason) override;
     const std::string& getParsingStartTag() const override {
         static const std::string toolCallStartTag = "[TOOL_CALLS]";

src/llm/io_processing/output_parser.cpp

@@ -143,8 +143,9 @@ std::optional<rapidjson::Document> OutputParser::parseReasoningChunk(ov::genai::
     return result;
 }
 
-OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string toolParserName, const std::string reasoningParserName) :
+OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap) :
     tokenizer(tokenizer) {
+    SPDLOG_ERROR("OutputParser created with toolNameSchemaMap of size: {}", toolNameSchemaMap.size());
     if (toolParserName == "llama3") {
         toolParser = std::make_unique<Llama3ToolParser>(tokenizer);
     } else if (toolParserName == "hermes3") {
@@ -156,7 +157,7 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to
     } else if (toolParserName == "gptoss") {
         toolParser = std::make_unique<GptOssToolParser>(tokenizer);
     } else if (toolParserName == "qwen3coder") {
-        toolParser = std::make_unique<Qwen3CoderToolParser>(tokenizer);
+        toolParser = std::make_unique<Qwen3CoderToolParser>(tokenizer, toolNameSchemaMap);
     } else if (!toolParserName.empty()) {
         throw std::runtime_error("Unsupported tool parser: " + toolParserName);
     }
@@ -201,7 +202,7 @@ std::string OutputParser::getToolParserStartTag() const {
     }
 }
 
-ParsedOutput OutputParser::parse(const std::vector<int64_t>& generatedTokens, const bool toolsAvailable, const ToolsSchemas_t& toolNameSchemaMap) {
+ParsedOutput OutputParser::parse(const std::vector<int64_t>& generatedTokens, const bool toolsAvailable) {
     // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure.
     // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens.
     // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output.
@@ -212,11 +213,11 @@ ParsedOutput OutputParser::parse(const std::vector<int64_t>& generatedTokens, co
     ParsedOutput parsedOutput;
     parsedOutput.content = tokenizer.decode(generatedTokens);
     if (reasoningParser) {
-        reasoningParser->parse(parsedOutput, generatedTokens, toolNameSchemaMap);
+        reasoningParser->parse(parsedOutput, generatedTokens);
     }
     // We run tool parser only if the parser is available and tools have been provided in the request.
     if (toolParser && toolsAvailable) {
-        toolParser->parse(parsedOutput, generatedTokens, toolNameSchemaMap);
+        toolParser->parse(parsedOutput, generatedTokens);
     }
     return parsedOutput;
 }
@@ -231,7 +232,7 @@ std::optional<rapidjson::Document> OutputParser::parseChunk(const std::string& c
     */
 
     bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTag().empty() && !reasoningParser->getParsingEndTag().empty();
-    bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTag().empty();
+    bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTag().empty(); // FIXME why not check for parsingEntTag not empty?
     bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable;
 
     if (applyToolParser && toolParser->isImmediateParsingEnabled() && processingPhase == UNKNOWN) {