[Sync] update github token (#475)

open-compass · Oct 13, 2023 · fbf5089 · fbf5089
1 parent 362c33d
commit fbf5089
Show file tree

Hide file tree

Showing 24 changed files with 1,678 additions and 104 deletions.
diff --git a/.codespellrc b/.codespellrc
@@ -2,4 +2,4 @@
 skip = *.ipynb
 count =
 quiet-level = 3
-ignore-words-list = nd, ans, ques, rouge, softwares
+ignore-words-list = nd, ans, ques, rouge, softwares, wit
diff --git a/configs/datasets/subjectivity_cmp/subjectivity_cmp.py b/configs/datasets/subjectivity_cmp/subjectivity_cmp.py
@@ -0,0 +1,61 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.datasets.subjectivity_cmp import SubjectivityCmpDataset
+
+subjectivity_reader_cfg = dict(
+    input_columns=['question', 'index', 'reference_answer', 'evaluating_guidance', 'capability', 'prompt'],
+    output_column=None,
+    train_split='test')
+
+subjectivity_all_sets = [
+    "sub_test",
+]
+
+subjectivity_datasets = []
+
+for _name in subjectivity_all_sets:
+    subjectivity_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt="{question}"
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer),
+        )
+
+    subjectivity_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            cmp_order='both',
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    begin=[
+                        dict(
+                            role="SYSTEM",
+                            fallback_role="HUMAN",
+                            prompt="{prompt}"
+                        ),
+                    ],
+                    round=[dict(role="HUMAN",
+                                prompt="回答 1: <回答 1 开始> {prediction} <回答 1 结束>\n回答 2: <回答 2 开始> {prediction2} <回答 2 结束>\n")]))),
+        pred_role="BOT",
+    )
+
+    subjectivity_datasets.append(
+        dict(
+            abbr=f"{_name}",
+            type=SubjectivityCmpDataset,
+            path="./data/subjectivity/",
+            name=_name,
+            reader_cfg=subjectivity_reader_cfg,
+            infer_cfg=subjectivity_infer_cfg,
+            eval_cfg=subjectivity_eval_cfg
+        ))
diff --git a/configs/subjective_infer.py b/configs/subjective_infer.py
@@ -0,0 +1,122 @@
+from mmengine.config import read_base
+with read_base():
+    from .datasets.subjectivity_cmp.subjectivity_cmp import subjectivity_datasets
+    from .summarizers.subjective import summarizer
+
+datasets = [*subjectivity_datasets]
+
+from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAI
+from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks.subjective_eval import SubjectiveEvalTask
+
+_meta_template = dict(
+    round=[
+        dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
+        dict(
+            role="BOT",
+            begin="\n<|im_start|>assistant\n",
+            end='<|im_end|>',
+            generate=True),
+    ], )
+
+_meta_template2 = dict(
+    round=[
+        dict(role='HUMAN', begin='<|User|>:', end='<eoh>\n'),
+        dict(role='BOT', begin='<|Bot|>:', end='<eoa>\n', generate=True),
+    ], )
+
+models = [
+    dict(
+        type=HuggingFace,
+        abbr='chatglm2-6b-hf',
+        path='THUDM/chatglm2-6b',
+        tokenizer_path='THUDM/chatglm2-6b',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            revision='b1502f4f75c71499a3d566b14463edd62620ce9f'),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(
+            trust_remote_code=True,
+            device_map='auto',
+            revision='b1502f4f75c71499a3d566b14463edd62620ce9f'),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    ),
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='qwen-7b-chat-hf',
+        path="/mnt/petrelfs/share_data/duanhaodong/Qwen-7B-Chat",
+        tokenizer_path='/mnt/petrelfs/share_data/duanhaodong/Qwen-7B-Chat',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            use_fast=False,
+        ),
+        pad_token_id=151643,
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        meta_template=_meta_template,
+        model_kwargs=dict(device_map='auto', trust_remote_code=True),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    ),
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='internlm-chat-7b-hf',
+        path="internlm/internlm-chat-7b",
+        tokenizer_path='internlm/internlm-chat-7b',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            use_fast=False,
+            trust_remote_code=True,
+            revision="ed5e35564ac836710817c51e8e8d0a5d4ff03102"),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=8,
+        meta_template=_meta_template2,
+        model_kwargs=dict(
+            trust_remote_code=True,
+            device_map='auto',
+            revision="ed5e35564ac836710817c51e8e8d0a5d4ff03102"),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
+]
+
+api_meta_template = dict(
+    round=[
+        dict(role='HUMAN', api_role='HUMAN'),
+        dict(role='BOT', api_role='BOT', generate=True)
+    ],
+    reserved_roles=[
+        dict(role='SYSTEM', api_role='SYSTEM'),
+    ],
+)
+
+eval = dict(
+    partitioner=dict(
+        type=SubjectiveNaivePartitioner,
+        mode='all',  # 新参数
+    ),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=2,  # 支持并行比较
+        task=dict(
+            type=SubjectiveEvalTask,  # 新 task，用来读入一对 model 的输入
+            judge_cfg=dict(
+                abbr='GPT4',
+                type=OpenAI,
+                path='gpt-4-0613',
+                key='ENV',
+                meta_template=api_meta_template,
+                query_per_second=1,
+                max_out_len=2048,
+                max_seq_len=2048,
+                batch_size=2),
+        )),
+)
diff --git a/configs/summarizers/example.py b/configs/summarizers/example.py
@@ -14,8 +14,4 @@
 
 summarizer = dict(
     summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []),
-    prompt_db=dict(
-        database_path='configs/datasets/log.json',
-        config_dir='configs/datasets',
-        blacklist='.promptignore')
 )
diff --git a/configs/summarizers/leaderboard.py b/configs/summarizers/leaderboard.py
@@ -82,8 +82,4 @@
     ],
     summary_groups=sum(
         [v for k, v in locals().items() if k.endswith("_summary_groups")], []),
-    prompt_db=dict(
-        database_path='configs/datasets/log.json',
-        config_dir='configs/datasets',
-        blacklist='.promptignore'),
 )
diff --git a/configs/summarizers/leval.py b/configs/summarizers/leval.py
@@ -22,8 +22,4 @@
         'LEval_tvshow_summ'
     ],
     summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []),
-    prompt_db=dict(
-        database_path='configs/datasets/log.json',
-        config_dir='configs/datasets',
-        blacklist='.promptignore'),
 )
diff --git a/configs/summarizers/longbench.py b/configs/summarizers/longbench.py
@@ -29,8 +29,4 @@
         'LongBench_repobench-p',
     ],
     summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []),
-    prompt_db=dict(
-        database_path='configs/datasets/log.json',
-        config_dir='configs/datasets',
-        blacklist='.promptignore'),
 )
diff --git a/configs/summarizers/medium.py b/configs/summarizers/medium.py
@@ -101,8 +101,4 @@
     ],
     summary_groups=sum(
         [v for k, v in locals().items() if k.endswith("_summary_groups")], []),
-    prompt_db=dict(
-        database_path='configs/datasets/log.json',
-        config_dir='configs/datasets',
-        blacklist='.promptignore'),
 )
diff --git a/configs/summarizers/small.py b/configs/summarizers/small.py
@@ -60,8 +60,4 @@
         'crows_pairs',
     ],
     summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []),
-    prompt_db=dict(
-        database_path='configs/datasets/log.json',
-        config_dir='configs/datasets',
-        blacklist='.promptignore'),
 )
diff --git a/configs/summarizers/subjective.py b/configs/summarizers/subjective.py
@@ -0,0 +1,5 @@
+from opencompass.summarizers import SubjectiveSummarizer
+
+summarizer = dict(
+    type=SubjectiveSummarizer
+)
diff --git a/opencompass/datasets/lmeval.py b/opencompass/datasets/lmeval.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
 from datasets import Dataset, DatasetDict
 
 from opencompass.datasets import BaseDataset
@@ -10,8 +8,6 @@ class LMEvalDataset(BaseDataset):
     OpenCompass's internal use."""
 
     @staticmethod
-    def load(predictions: List, references: Optional[List] = None):
-        content = {'prediction': predictions}
-        if references:
-            content['reference'] = references
+    def load(**kwargs):
+        content = {k: v for k, v in kwargs.items() if v}
         return DatasetDict(dict(test=Dataset.from_dict(content)))