BUG: Fix missing qwen 1.5 7b gguf (#1027)

xorbitsai · Feb 22, 2024 · 79e2d06 · 79e2d06
1 parent 9efc998
commit 79e2d06
Show file tree

Hide file tree

Showing 2 changed files with 93 additions and 12 deletions.
diff --git a/xinference/model/llm/llm_family.json b/xinference/model/llm/llm_family.json
@@ -1514,10 +1514,33 @@
         ],
         "model_id": "Qwen/Qwen1.5-72B-Chat-AWQ"
       },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
+        ],
+        "model_id": "Qwen/Qwen1.5-0.5B-Chat-GGUF",
+        "model_file_name_template": "qwen1_5-0_5b-chat-{quantization}.gguf"
+      },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": "1_8",
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-1.8B-Chat-GGUF",
@@ -1527,6 +1550,13 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 4,
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-4B-Chat-GGUF",
@@ -1536,7 +1566,14 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 7,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-7B-Chat-GGUF",
         "model_file_name_template": "qwen1_5-7b-chat-{quantization}.gguf"
@@ -1545,7 +1582,14 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 14,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-14B-Chat-GGUF",
         "model_file_name_template": "qwen1_5-14b-chat-{quantization}.gguf"
@@ -1554,7 +1598,8 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 72,
         "quantizations": [
-          "q2_k"
+          "q2_k",
+          "q3_k_m"
         ],
         "model_id": "Qwen/Qwen1.5-72B-Chat-GGUF",
         "model_file_name_template": "qwen1_5-72b-chat-{quantization}.gguf"

diff --git a/xinference/model/llm/llm_family_modelscope.json b/xinference/model/llm/llm_family_modelscope.json
@@ -1821,61 +1821,97 @@
         "model_format": "ggufv2",
         "model_size_in_billions": "0_5",
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-0.5B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-0.5b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-0_5b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": "1_8",
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-1.8B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-1_8b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-1_8b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 4,
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-4B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-4b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-4b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 7,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-7B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-7b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-7b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 14,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-14B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-14b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-14b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 72,
         "quantizations": [
-          "q2_k"
+          "q2_k",
+          "q3_k_m"
         ],
         "model_id": "qwen/Qwen1.5-72B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-72b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-72b-chat-{quantization}.gguf"
       }
     ],
     "prompt_style": {