Merge remote-tracking branch 'origin/main'

# Conflicts: # doc/source/models/builtin/audio/index.rst # xinference/core/worker.py # xinference/deploy/docker/requirements.txt # xinference/deploy/docker/requirements_cpu.txt # xinference/model/audio/core.py # xinference/model/llm/llm_family.json # xinference/model/llm/lmdeploy/core.py # xinference/model/llm/transformers/intern_vl.py # xinference/model/llm/utils.py # xinference/model/llm/vllm/core.py
amumu96 · Nov 25, 2024 · 6a1a865 · 6a1a865
2 parents 88654d7 + bd599b2
commit 6a1a865
Show file tree

Hide file tree

Showing 338 changed files with 129,055 additions and 9,939 deletions.
diff --git a/.github/workflows/docker-cd.yaml b/.github/workflows/docker-cd.yaml
@@ -14,6 +14,7 @@ concurrency:
 
 jobs:
   build:
+    timeout-minutes: 120
     runs-on: self-hosted
     strategy:
       matrix:
@@ -85,10 +86,6 @@ jobs:
         env:
           DOCKER_ORG: registry.cn-hangzhou.aliyuncs.com/xprobe_xinference
         run: |
-          docker tag "xprobe/xinference:${XINFERENCE_IMAGE_TAG}" "${DOCKER_ORG}/xinference:${XINFERENCE_IMAGE_TAG}"
-          docker push "${DOCKER_ORG}/xinference:${XINFERENCE_IMAGE_TAG}"
-          docker tag "xprobe/xinference:${XINFERENCE_IMAGE_TAG}-cpu" "${DOCKER_ORG}/xinference:${XINFERENCE_IMAGE_TAG}-cpu"
-          docker push "${DOCKER_ORG}/xinference:${XINFERENCE_IMAGE_TAG}-cpu"
           if [[ -n "$XINFERENCE_GIT_TAG" ]]; then
             docker tag "xprobe/xinference:${XINFERENCE_GIT_TAG}" "$DOCKER_ORG/xinference:latest"
             docker push "$DOCKER_ORG/xinference:latest"

diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
@@ -135,14 +135,17 @@ jobs:
           pip install tensorizer
           pip install eva-decord
           pip install jj-pytorchvideo
+          pip install qwen-vl-utils
+          pip install datamodel_code_generator
+          pip install jsonschema
         working-directory: .
 
       - name: Test with pytest
         env:
           MODULE: ${{ matrix.module }}
         run: |
           if [ "$MODULE" == "gpu" ]; then
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "openai>1,<1.40"
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "openai>1"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U modelscope
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U sse_starlette
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U xoscar
@@ -154,37 +157,43 @@ jobs:
             ${{ env.SELF_HOST_CONDA }} install -c conda-forge pynini=2.1.5
             ${{ env.SELF_HOST_CONDA }} install -c conda-forge "ffmpeg<7"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U funasr
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U nemo_text_processing
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U nemo_text_processing<1.1.0
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U omegaconf~=2.3.0
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U WeTextProcessing
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U WeTextProcessing<1.0.4
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U librosa
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U xxhash
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "ChatTTS>0.1"
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "ChatTTS>=0.2"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U HyperPyYAML
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U matcha-tts
+            ${{ env.SELF_HOST_PYTHON }} -m pip uninstall -y matcha-tts
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U onnxruntime-gpu==1.16.0; sys_platform == 'linux'
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U openai-whisper
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U "torch==2.3.1" "torchaudio==2.3.1"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U "loguru"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U "natsort"
             ${{ env.SELF_HOST_PYTHON }} -m pip install -U "loralib"
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "opencc==1.1.6"
-            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "faster_whisper"
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U "ormsgpack"
+            ${{ env.SELF_HOST_PYTHON }} -m pip uninstall -y opencc
+            ${{ env.SELF_HOST_PYTHON }} -m pip uninstall -y "faster_whisper"
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U accelerate
+            ${{ env.SELF_HOST_PYTHON }} -m pip install -U verovio
             ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
-              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/image/tests/test_stable_diffusion.py
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/image/tests/test_stable_diffusion.py && \
             ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
-              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_whisper.py
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/image/tests/test_got_ocr2.py && \
             ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
-              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_funasr.py
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_whisper.py && \
             ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
-              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_chattts.py
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_funasr.py && \
             ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
-              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_cosyvoice.py
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_chattts.py && \
+            ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
+              -W ignore::PendingDeprecationWarning \
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_cosyvoice.py && \
             ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
               --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_fish_speech.py
@@ -198,6 +207,6 @@ jobs:
               --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/client/tests/test_client.py
             pytest --timeout=1500 \
               -W ignore::PendingDeprecationWarning \
-              --cov-config=setup.cfg --cov-report=xml --cov=xinference --ignore xinference/client/tests/test_client.py --ignore xinference/model/image/tests/test_stable_diffusion.py --ignore xinference/model/audio/tests xinference
+              --cov-config=setup.cfg --cov-report=xml --cov=xinference --ignore xinference/client/tests/test_client.py --ignore xinference/model/image/tests/test_stable_diffusion.py --ignore xinference/model/image/tests/test_got_ocr2.py --ignore xinference/model/audio/tests xinference
           fi
         working-directory: .
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -12,4 +12,5 @@ global-exclude conftest.py
 include xinference/locale/*.json
 include xinference/model/llm/*.json
 include xinference/model/embedding/*.json
+graft xinference/thirdparty
 global-include xinference/web/ui/build/**/*
diff --git a/README.md b/README.md
@@ -3,13 +3,25 @@
 
 # Xorbits Inference: Model Serving Made Easy 🤖
 
+<p align="center">
+  <a href="https://inference.top/">Xinference Cloud</a> ·
+  <a href="https://github.com/xorbitsai/enterprise-docs/blob/main/README.md">Xinference Enterprise</a> ·
+  <a href="https://inference.readthedocs.io/en/latest/getting_started/installation.html#installation">Self-hosting</a> ·
+  <a href="https://inference.readthedocs.io/">Documentation</a>
+</p>
+
 [![PyPI Latest Release](https://img.shields.io/pypi/v/xinference.svg?style=for-the-badge)](https://pypi.org/project/xinference/)
 [![License](https://img.shields.io/pypi/l/xinference.svg?style=for-the-badge)](https://github.com/xorbitsai/inference/blob/main/LICENSE)
 [![Build Status](https://img.shields.io/github/actions/workflow/status/xorbitsai/inference/python.yaml?branch=main&style=for-the-badge&label=GITHUB%20ACTIONS&logo=github)](https://actions-badge.atrox.dev/xorbitsai/inference/goto?ref=main)
 [![Slack](https://img.shields.io/badge/join_Slack-781FF5.svg?logo=slack&style=for-the-badge)](https://join.slack.com/t/xorbitsio/shared_invite/zt-1o3z9ucdh-RbfhbPVpx7prOVdM1CAuxg)
 [![Twitter](https://img.shields.io/twitter/follow/xorbitsio?logo=x&style=for-the-badge)](https://twitter.com/xorbitsio)
 
-English | [中文介绍](README_zh_CN.md) | [日本語](README_ja_JP.md)
+<p align="center">
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-454545?style=for-the-badge"></a>
+  <a href="./README_zh_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/中文介绍-d9d9d9?style=for-the-badge"></a>
+  <a href="./README_ja_JP.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9?style=for-the-badge"></a>
+</p>
+
 </div>
 <br />
 
@@ -34,14 +46,14 @@ potential of cutting-edge AI models.
 - Support speech recognition model: [#929](https://github.com/xorbitsai/inference/pull/929)
 - Metrics support: [#906](https://github.com/xorbitsai/inference/pull/906)
 ### New Models
+- Built-in support for [Qwen 2.5 Series](https://qwenlm.github.io/blog/qwen2.5/): [#2325](https://github.com/xorbitsai/inference/pull/2325)
+- Built-in support for [Fish Speech V1.4](https://huggingface.co/fishaudio/fish-speech-1.4): [#2295](https://github.com/xorbitsai/inference/pull/2295)
+- Built-in support for [DeepSeek-V2.5](https://huggingface.co/deepseek-ai/DeepSeek-V2.5): [#2292](https://github.com/xorbitsai/inference/pull/2292)
+- Built-in support for [Qwen2-Audio](https://github.com/QwenLM/Qwen2-Audio): [#2271](https://github.com/xorbitsai/inference/pull/2271)
+- Built-in support for [Qwen2-vl-instruct](https://github.com/QwenLM/Qwen2-VL): [#2205](https://github.com/xorbitsai/inference/pull/2205)
+- Built-in support for [MiniCPM3-4B](https://huggingface.co/openbmb/MiniCPM3-4B): [#2263](https://github.com/xorbitsai/inference/pull/2263)
 - Built-in support for [CogVideoX](https://github.com/THUDM/CogVideo): [#2049](https://github.com/xorbitsai/inference/pull/2049)
 - Built-in support for [flux.1-schnell & flux.1-dev](https://www.basedlabs.ai/tools/flux1): [#2007](https://github.com/xorbitsai/inference/pull/2007)
-- Built-in support for [MiniCPM-V 2.6](https://github.com/OpenBMB/MiniCPM-V): [#2031](https://github.com/xorbitsai/inference/pull/2031)
-- Built-in support for [Kolors](https://huggingface.co/Kwai-Kolors/Kolors): [#2028](https://github.com/xorbitsai/inference/pull/2028)
-- Built-in support for [SenseVoice](https://github.com/FunAudioLLM/SenseVoice): [#2008](https://github.com/xorbitsai/inference/pull/2008)
-- Built-in support for [Mistral Large 2](https://mistral.ai/news/mistral-large-2407/): [#1944](https://github.com/xorbitsai/inference/pull/1944)
-- Built-in support for [llama3.1](https://ai.meta.com/blog/meta-llama-3-1/): [#1932](https://github.com/xorbitsai/inference/pull/1932)
-- Built-in support for [Mistral Nemo](https://mistral.ai/news/mistral-nemo/): [#1936](https://github.com/xorbitsai/inference/pull/1936)
 ### Integrations
 - [Dify](https://docs.dify.ai/advanced/model-configuration/xinference): an LLMOps platform that enables developers (and even non-developers) to quickly build useful applications based on large language models, ensuring they are visual, operable, and improvable.
 - [FastGPT](https://github.com/labring/FastGPT): a knowledge-based platform built on the LLM, offers out-of-the-box data processing and model invocation capabilities, allows for workflow orchestration through Flow visualization.
@@ -85,9 +97,25 @@ with popular third-party libraries including [LangChain](https://python.langchai
 | Audio Models                                   | ✅ | ❌ | ❌ | ❌ |
 | More OpenAI Functionalities (Function Calling) | ✅ | ❌ | ❌ | ❌ |
 
-## Getting Started
+## Using Xinference
+
+- **Cloud </br>**
+We host a [Xinference Cloud](https://inference.top) service for anyone to try with zero setup. 
+
+- **Self-hosting Xinference Community Edition</br>**
+Quickly get Xinference running in your environment with this [starter guide](#getting-started).
+Use our [documentation](https://inference.readthedocs.io/) for further references and more in-depth instructions.
+
+- **Xinference for enterprise / organizations</br>**
+We provide additional enterprise-centric features. [send us an email](mailto:[email protected]?subject=[GitHub]Business%20License%20Inquiry) to discuss enterprise needs. </br>
 
-**Please give us a star before you begin, and you'll receive instant notifications for every new release on GitHub!**
+## Staying Ahead
+
+Star Xinference on GitHub and be instantly notified of new releases.
+
+![star-us](assets/stay_ahead.gif)
+
+## Getting Started
 
 * [Docs](https://inference.readthedocs.io/en/latest/index.html)
 * [Built-in Models](https://inference.readthedocs.io/en/latest/models/builtin/index.html)
@@ -157,3 +185,7 @@ Once Xinference is running, there are multiple ways you can try it: via the web
 <a href="https://github.com/xorbitsai/inference/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=xorbitsai/inference" />
 </a>
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=xorbitsai/inference&type=Date)](https://star-history.com/#xorbitsai/inference&Date)
diff --git a/README_ja_JP.md b/README_ja_JP.md
@@ -9,7 +9,11 @@
 [![Slack](https://img.shields.io/badge/join_Slack-781FF5.svg?logo=slack&style=for-the-badge)](https://join.slack.com/t/xorbitsio/shared_invite/zt-1o3z9ucdh-RbfhbPVpx7prOVdM1CAuxg)
 [![Twitter](https://img.shields.io/twitter/follow/xorbitsio?logo=x&style=for-the-badge)](https://twitter.com/xorbitsio)
 
-[English](README.md) | [中文介绍](README_zh_CN.md) | 日本語
+<p align="center">
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9?style=for-the-badge"></a>
+  <a href="./README_zh_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/中文介绍-d9d9d9?style=for-the-badge"></a>
+  <a href="./README_ja_JP.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-454545?style=for-the-badge"></a>
+</p>
 </div>
 <br />
 

diff --git a/README_zh_CN.md b/README_zh_CN.md
@@ -3,13 +3,24 @@
 
 # Xorbits Inference：模型推理， 轻而易举 🤖
 
+<p align="center">
+  <a href="https://inference.top/">Xinference 云服务</a> ·
+  <a href="https://github.com/xorbitsai/enterprise-docs/blob/main/README_zh_CN.md">Xinference 企业版</a> ·
+  <a href="https://inference.readthedocs.io/zh-cn/latest/getting_started/installation.html#installation">自托管</a> ·
+  <a href="https://inference.readthedocs.io/">文档</a>
+</p>
+
 [![PyPI Latest Release](https://img.shields.io/pypi/v/xinference.svg?style=for-the-badge)](https://pypi.org/project/xinference/)
 [![License](https://img.shields.io/pypi/l/xinference.svg?style=for-the-badge)](https://github.com/xorbitsai/inference/blob/main/LICENSE)
 [![Build Status](https://img.shields.io/github/actions/workflow/status/xorbitsai/inference/python.yaml?branch=main&style=for-the-badge&label=GITHUB%20ACTIONS&logo=github)](https://actions-badge.atrox.dev/xorbitsai/inference/goto?ref=main)
 [![WeChat](https://img.shields.io/badge/添加微信小助手-07C160?style=for-the-badge&logo=wechat&logoColor=white)](https://xorbits.cn/assets/images/wechat_work_qr.png)
 [![Zhihu](https://img.shields.io/static/v1?style=for-the-badge&message=未来速度&color=0084FF&logo=Zhihu&logoColor=FFFFFF&label=)](https://www.zhihu.com/org/xorbits)
 
-[English](README.md) | 中文介绍 | [日本語](README_ja_JP.md)
+<p align="center">
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9?style=for-the-badge"></a>
+  <a href="./README_zh_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/中文介绍-454545?style=for-the-badge"></a>
+  <a href="./README_ja_JP.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9?style=for-the-badge"></a>
+</p>
 </div>
 <br />
 
@@ -31,14 +42,14 @@ Xorbits Inference（Xinference）是一个性能强大且功能全面的分布
 - 支持语音识别模型: [#929](https://github.com/xorbitsai/inference/pull/929)
 - 增加 Metrics 统计信息: [#906](https://github.com/xorbitsai/inference/pull/906)
 ### 新模型
+- 内置 [Qwen 2.5 Series](https://qwenlm.github.io/blog/qwen2.5/): [#2325](https://github.com/xorbitsai/inference/pull/2325)
+- 内置 [Fish Speech V1.4](https://huggingface.co/fishaudio/fish-speech-1.4): [#2295](https://github.com/xorbitsai/inference/pull/2295)
+- 内置 [DeepSeek-V2.5](https://huggingface.co/deepseek-ai/DeepSeek-V2.5): [#2292](https://github.com/xorbitsai/inference/pull/2292)
+- 内置 [Qwen2-Audio](https://github.com/QwenLM/Qwen2-Audio): [#2271](https://github.com/xorbitsai/inference/pull/2271)
+- 内置 [Qwen2-vl-instruct](https://github.com/QwenLM/Qwen2-VL): [#2205](https://github.com/xorbitsai/inference/pull/2205)
+- 内置 [MiniCPM3-4B](https://huggingface.co/openbmb/MiniCPM3-4B): [#2263](https://github.com/xorbitsai/inference/pull/2263)
 - 内置 [CogVideoX](https://github.com/THUDM/CogVideo): [#2049](https://github.com/xorbitsai/inference/pull/2049)
 - 内置 [flux.1-schnell & flux.1-dev](https://www.basedlabs.ai/tools/flux1): [#2007](https://github.com/xorbitsai/inference/pull/2007)
-- 内置 [MiniCPM-V 2.6](https://github.com/OpenBMB/MiniCPM-V): [#2031](https://github.com/xorbitsai/inference/pull/2031)
-- 内置 [Kolors](https://huggingface.co/Kwai-Kolors/Kolors): [#2028](https://github.com/xorbitsai/inference/pull/2028)
-- 内置 [SenseVoice](https://github.com/FunAudioLLM/SenseVoice): [#2008](https://github.com/xorbitsai/inference/pull/2008)
-- 内置 [Mistral Large 2](https://mistral.ai/news/mistral-large-2407/): [#1944](https://github.com/xorbitsai/inference/pull/1944)
-- 内置 [llama3.1](https://ai.meta.com/blog/meta-llama-3-1/): [#1932](https://github.com/xorbitsai/inference/pull/1932)
-- 内置 [Mistral Nemo](https://mistral.ai/news/mistral-nemo/): [#1936](https://github.com/xorbitsai/inference/pull/1936)
 ### 集成
 - [FastGPT](https://doc.fastai.site/docs/development/custom-models/xinference/)：一个基于 LLM 大模型的开源 AI 知识库构建平台。提供了开箱即用的数据处理、模型调用、RAG 检索、可视化 AI 工作流编排等能力，帮助您轻松实现复杂的问答场景。
 - [Dify](https://docs.dify.ai/advanced/model-configuration/xinference): 一个涵盖了大型语言模型开发、部署、维护和优化的 LLMOps 平台。
@@ -72,10 +83,26 @@ Xorbits Inference（Xinference）是一个性能强大且功能全面的分布
 | 语音识别模型                  | ✅ | ❌ | ❌ | ❌ |
 | 更多 OpenAI 功能 (函数调用)     | ✅ | ❌ | ❌ | ❌ |
 
+## 使用 Xinference
 
-## 入门指南
+- **云 </br>**
+我们提供 [Xinference 云服务](https://inference.top)，无需任何设置。
+
+- **自托管 Xinference 社区版</br>**
+使用 [入门指南](#getting-started) 快速在你自己的环境中运行 Xinference。
+参考 [文档](https://inference.readthedocs.io/zh-cn) 以获得参考和更多说明。
+
+- **面向企业/组织的 Xinference 版本</br>**
+我们提供额外的面向企业的功能。 [通过企业微信联系](https://xorbits.cn/assets/images/wechat_work_qr.png)
+或 [提交表单](https://w8v6grm432.feishu.cn/share/base/form/shrcn9u1EBXQxmGMqILEjguuGoh) 讨论企业需求。 </br>
+
+## 保持领先
 
-**在开始之前，请给我们一个星标，这样你就可以在 GitHub 上及时收到每个新版本的通知！**
+在 GitHub 上给 Xinference Star，并立即收到新版本的通知。
+
+![star-us](assets/stay_ahead.gif)
+
+## 入门指南
 
 * [文档](https://inference.readthedocs.io/zh-cn/latest/index.html)
 * [内置模型](https://inference.readthedocs.io/zh-cn/latest/models/builtin/index.html)
@@ -141,4 +168,8 @@ $ xinference-local
 
 <a href="https://github.com/xorbitsai/inference/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=xorbitsai/inference" />
-</a>
+</a>
+
+## Star 历史
+
+[![Star History Chart](https://api.star-history.com/svg?repos=xorbitsai/inference&type=Date)](https://star-history.com/#xorbitsai/inference&Date)
diff --git a/assets/stay_ahead.gif b/assets/stay_ahead.gif