ENH: Add gguf format of codellama-instruct (#567)

xorbitsai · Oct 20, 2023 · a6bf734 · a6bf734
1 parent 14fc29c
commit a6bf734
Showing 1 changed file with 33 additions and 0 deletions.
diff --git a/xinference/model/llm/llm_family_modelscope.json b/xinference/model/llm/llm_family_modelscope.json
@@ -571,6 +571,39 @@
         "model_hub": "modelscope",
         "model_id": "AI-ModelScope/CodeLlama-34b-Instruct-hf",
         "model_revision": "v1.0.2"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "Q4_K_M"
+        ],
+        "model_hub": "modelscope",
+        "model_id": "Xorbits/CodeLlama-7B-Instruct-GGUF",
+        "model_file_name_template": "codellama-7b-instruct.{quantization}.gguf",
+        "model_revision": "v0.0.1"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 13,
+        "quantizations": [
+          "Q4_K_M"
+        ],
+        "model_hub": "modelscope",
+        "model_id": "Xorbits/CodeLlama-13B-Instruct-GGUF",
+        "model_file_name_template": "codellama-13b-instruct.{quantization}.gguf",
+        "model_revision": "v0.0.1"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 34,
+        "quantizations": [
+          "Q4_K_M"
+        ],
+        "model_hub": "modelscope",
+        "model_id": "Xorbits/CodeLlama-34B-Instruct-GGUF",
+        "model_file_name_template": "codellama-34b-instruct.{quantization}.gguf",
+        "model_revision": "v0.1.0"
       }
     ],
     "prompt_style": {