支持 lmcache 升级sglang 为最新，vllm后端默认开启 lmcache

Author: shell-nlp

gpt_server/model_backend/sglang_backend.py

@@ -1,9 +1,7 @@
 import asyncio
 import base64
 from io import BytesIO
-import os
 from typing import Any, Dict, AsyncGenerator, List, Optional
-from fastchat.utils import is_partial_stop
 from gpt_server.model_backend.base import ModelBackend
 from loguru import logger
 from PIL import Image

gpt_server/model_backend/vllm_backend.py

@@ -8,9 +8,7 @@
 from vllm.lora.request import LoRARequest
 from transformers import PreTrainedTokenizer
 from vllm.entrypoints.chat_utils import (
-    ConversationMessage,
     apply_hf_chat_template,
-    load_chat_template,
     parse_chat_messages_futures,
 )
 from gpt_server.settings import get_model_config
@@ -40,6 +38,7 @@ def __init__(self, model_path, tokenizer: PreTrainedTokenizer) -> None:
                         lora_local_path=lora_path,
                     )
                 )
+        from vllm.config.kv_transfer import KVTransferConfig
 
         self.engine_args = AsyncEngineArgs(
             model_path,
@@ -53,6 +52,10 @@ def __init__(self, model_path, tokenizer: PreTrainedTokenizer) -> None:
             dtype=model_config.dtype,
             max_model_len=model_config.max_model_len,
             guided_decoding_backend="xgrammar",
+            # 支持LMCache的KV传输
+            kv_transfer_config=KVTransferConfig(
+                kv_connector="LMCacheConnectorV1", kv_role="kv_both"
+            ),
         )
         self.engine = AsyncLLMEngine.from_engine_args(self.engine_args)
         self.tokenizer = tokenizer

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gpt_server"
-version = "0.6.6"
+version = "0.6.7"
 description = "gpt_server是一个用于生产级部署LLMs、Embedding、Reranker、ASR和TTS的开源框架。"
 readme = "README.md"
 license = { text = "Apache 2.0" }
@@ -14,7 +14,7 @@ dependencies = [
     "infinity-emb[all]==0.0.77",
     "lmdeploy==0.10.2",
     "loguru>=0.7.2",
-    "openai==1.99.1",
+    "openai==2.6.1",
     "setuptools==75.2.0",
     "streamlit>=1.50.0",
     "torch==2.8.0",
@@ -25,12 +25,13 @@ dependencies = [
     "modelscope>=1.31.0",
     "edge-tts>=7.0.0",
     "funasr>=1.2.6",
-    "sglang[all]>=0.5.4",
+    "sglang[all]>=0.5.5",
     "flashinfer-python",
     "flashtts>=0.1.7",
     "diffusers>=0.35.2",
     "sqlmodel>=0.0.27",
     "autoawq>=0.2.9",
+    "lmcache>=0.3.9.post1",
 ]
 
 [tool.uv]