[Feaure] Add new models: baichuan2, tigerbot, vicuna v1.5 (#373)

* add bag of new models: baichuan2, tigerbot, vicuna v1.5

* update

* re-organize models

* update readme

* update

configs/eval_claude.py

@@ -8,7 +8,7 @@
     from .datasets.collections.chat_medium import datasets
     # and output the results in a choosen format
     from .summarizers.medium import summarizer
-    from .models.claude import models
+    from .models.claude.claude import models
 
 infer = dict(
     partitioner=dict(type=NaivePartitioner),

configs/eval_codegeex2.py

@@ -2,6 +2,6 @@
 
 with read_base():
     from .datasets.humanevalx.humanevalx_gen import humanevalx_datasets
-    from .models.hf_codegeex2_6b import models
+    from .models.codegeex2.hf_codegeex2_6b import models
 
-datasets = humanevalx_datasets
+datasets = humanevalx_datasets

configs/eval_demo.py

@@ -3,8 +3,8 @@
 with read_base():
     from .datasets.siqa.siqa_gen import siqa_datasets
     from .datasets.winograd.winograd_ppl import winograd_datasets
-    from .models.hf_opt_125m import opt125m
-    from .models.hf_opt_350m import opt350m
+    from .models.opt.hf_opt_125m import opt125m
+    from .models.opt.hf_opt_350m import opt350m
 
 datasets = [*siqa_datasets, *winograd_datasets]
 models = [opt125m, opt350m]

configs/eval_hf_llama_7b.py

@@ -2,7 +2,7 @@
 
 with read_base():
     from .datasets.collections.base_medium_llama import piqa_datasets, siqa_datasets
-    from .models.hf_llama_7b import models
+    from .models.hf_llama.hf_llama_7b import models
 
 
 datasets = [*piqa_datasets, *siqa_datasets]

configs/eval_internLM.py

@@ -4,6 +4,6 @@
     # choose a list of datasets
     from .datasets.collections.base_medium import datasets
     # choose a model of interest
-    from .models.internlm_7b import models
+    from .models.internlm.internlm_7b import models
     # and output the results in a choosen format
-    from .summarizers.medium import summarizer
+    from .summarizers.medium import summarizer

configs/eval_internlm_7b.py

@@ -4,6 +4,6 @@
     # choose a list of datasets
     from .datasets.collections.base_medium import datasets
     # choose a model of interest
-    from .models.hf_internlm_7b import models
+    from .models.hf_internlm.hf_internlm_7b import models
     # and output the results in a choosen format
     from .summarizers.medium import summarizer

configs/eval_llama2_7b.py

@@ -2,7 +2,7 @@
 
 with read_base():
     from .datasets.collections.base_medium_llama import piqa_datasets, siqa_datasets
-    from .models.llama2_7b import models
+    from .models.llama.llama2_7b import models
 
 
 datasets = [*piqa_datasets, *siqa_datasets]

configs/eval_qwen_7b.py

@@ -1,7 +1,7 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .models.hf_qwen_7b import models
+    from .models.qwen.hf_qwen_7b import models
     from .datasets.collections.leaderboard.qwen import datasets
     from .summarizers.leaderboard import summarizer
 

configs/eval_qwen_7b_chat.py

@@ -1,7 +1,7 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .models.hf_qwen_7b_chat import models
+    from .models.qwen.hf_qwen_7b_chat import models
     from .datasets.collections.leaderboard.qwen_chat import datasets
     from .summarizers.leaderboard import summarizer
 

configs/models/baichuan/hf_baichuan2_13b_base.py

@@ -0,0 +1,21 @@
+from opencompass.models import HuggingFaceCausalLM
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='baichuan2-13b-base-hf',
+        path="baichuan-inc/Baichuan2-13B-Base",
+        tokenizer_path='baichuan-inc/Baichuan2-13B-Base',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            use_fast=False,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(device_map='auto', trust_remote_code=True),
+        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
+]

configs/models/baichuan/hf_baichuan2_13b_chat.py

@@ -0,0 +1,29 @@
+from opencompass.models import HuggingFaceCausalLM
+
+_meta_template = dict(
+    round=[
+        dict(role='HUMAN', begin='<reserved_106>'),
+        dict(role='BOT', begin='<reserved_107>', generate=True),
+    ],
+)
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='baichuan2-13b-chat-hf',
+        path="baichuan-inc/Baichuan2-13B-Chat",
+        tokenizer_path='baichuan-inc/Baichuan2-13B-Chat',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            use_fast=False,
+        ),
+        meta_template=_meta_template,
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(device_map='auto', trust_remote_code=True),
+        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
+]

configs/models/baichuan/hf_baichuan2_7b_base.py

@@ -0,0 +1,21 @@
+from opencompass.models import HuggingFaceCausalLM
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='baichuan2-7b-base-hf',
+        path="baichuan-inc/Baichuan2-7B-Base",
+        tokenizer_path='baichuan-inc/Baichuan2-7B-Base',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            use_fast=False,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(device_map='auto', trust_remote_code=True),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
+]

configs/models/baichuan/hf_baichuan2_7b_chat.py

@@ -0,0 +1,29 @@
+from opencompass.models import HuggingFaceCausalLM
+
+_meta_template = dict(
+    round=[
+        dict(role='HUMAN', begin='<reserved_106>'),
+        dict(role='BOT', begin='<reserved_107>', generate=True),
+    ],
+)
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='baichuan2-7b-chat-hf',
+        path="baichuan-inc/Baichuan2-7B-Chat",
+        tokenizer_path='baichuan-inc/Baichuan2-7B-Chat',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            use_fast=False,
+        ),
+        meta_template=_meta_template,
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(device_map='auto', trust_remote_code=True),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
+]

configs/models/tigerbot/hf_tigerbot_13b_base_v1.py

@@ -0,0 +1,21 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='tigerbot-13b-base-v1-hf',
+        path='TigerResearch/tigerbot-13b-base-v1',
+        tokenizer_path='TigerResearch/tigerbot-13b-base-v1',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2, num_procs=1),
+    ),
+]

configs/models/tigerbot/hf_tigerbot_13b_base_v2.py

@@ -0,0 +1,21 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='tigerbot-13b-base-v2-hf',
+        path='TigerResearch/tigerbot-13b-base',
+        tokenizer_path='TigerResearch/tigerbot-13b-base',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2, num_procs=1),
+    ),
+]

configs/models/tigerbot/hf_tigerbot_13b_chat_v1.py

@@ -0,0 +1,29 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+_meta_template = dict(
+    round=[
+        dict(role='HUMAN', begin='\n\n### Instruction:\n'),
+        dict(role='BOT', begin='\n\n### Response:\n', generate=True),
+    ],
+)
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='tigerbot-13b-chat-v1-hf',
+        path="TigerResearch/tigerbot-13b-chat-v1",
+        tokenizer_path='TigerResearch/tigerbot-13b-chat-v1',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        meta_template=_meta_template,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
+]

configs/models/tigerbot/hf_tigerbot_13b_chat_v2.py

@@ -0,0 +1,29 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+_meta_template = dict(
+    round=[
+        dict(role='HUMAN', begin='\n\n### Instruction:\n'),
+        dict(role='BOT', begin='\n\n### Response:\n', generate=True),
+    ],
+)
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='tigerbot-13b-chat-v2-hf',
+        path="TigerResearch/tigerbot-13b-chat",
+        tokenizer_path='TigerResearch/tigerbot-13b-chat',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        meta_template=_meta_template,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
+]

configs/models/hf_tigerbot_7b_base.py → configs/models/tigerbot/hf_tigerbot_7b_base.py

@@ -15,7 +15,7 @@
         max_out_len=100,
         max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='5f34fd223586e9efb8eb0e3bc667f03581886992'),
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
         run_cfg=dict(num_gpus=1, num_procs=1),
     ),
 ]

configs/models/tigerbot/hf_tigerbot_7b_base_v3.py

@@ -0,0 +1,21 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='tigerbot-7b-base-v3-hf',
+        path='TigerResearch/tigerbot-7b-base',
+        tokenizer_path='TigerResearch/tigerbot-7b-base',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    ),
+]

configs/models/tigerbot/hf_tigerbot_7b_chat_v3.py

@@ -0,0 +1,29 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+_meta_template = dict(
+    round=[
+        dict(role='HUMAN', begin='\n\n### Instruction:\n'),
+        dict(role='BOT', begin='\n\n### Response:\n', generate=True),
+    ],
+)
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='tigerbot-7b-chat-v3-hf',
+        path="TigerResearch/tigerbot-7b-chat",
+        tokenizer_path='TigerResearch/tigerbot-7b-chat',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        meta_template=_meta_template,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
+]

configs/models/hf_tigerbot_7b_sft.py → configs/models/tigerbot/hf_tigerbot_7b_sft.py

@@ -23,7 +23,7 @@
         max_seq_len=2048,
         batch_size=8,
         meta_template=_meta_template,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='0ba4d6fc479bdedd6a3f8d4d3425025c5f501800'),
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
         run_cfg=dict(num_gpus=1, num_procs=1),
     )
 ]

configs/models/hf_vicuna_13b.py → configs/models/vicuna/hf_vicuna_13b_v13.py

@@ -4,7 +4,7 @@
 models = [
     dict(
         type=HuggingFaceCausalLM,
-        abbr='vicuna-13b-hf',
+        abbr='vicuna-13b-v1.3-hf',
         path="lmsys/vicuna-13b-v1.3",
         tokenizer_path='lmsys/vicuna-13b-v1.3',
         tokenizer_kwargs=dict(

configs/models/vicuna/hf_vicuna_13b_v15.py

@@ -0,0 +1,22 @@
+from opencompass.models import HuggingFaceCausalLM
+
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='vicuna-13b-v1.5-hf',
+        path="lmsys/vicuna-13b-v1.5",
+        tokenizer_path='lmsys/vicuna-13b-v1.5',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            use_fast=False,
+        ),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(device_map='auto'),
+        batch_padding=False, # if false, inference with for-loop without batch padding
+        run_cfg=dict(num_gpus=2, num_procs=1)
+    )
+]