Skip to content

Commit

Permalink
for code review
Browse files Browse the repository at this point in the history
  • Loading branch information
xingyuanbu committed May 30, 2024
1 parent 3ffd04f commit 1ac4513
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 17 deletions.
1 change: 0 additions & 1 deletion configs/datasets/subjective/multiround/mtbench101_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@


subjective_reader_cfg = dict(
# input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
input_columns=["dialogue","task","multi_id","turn_id","system_prompt",'prompt_template'],
output_column='judge',
)
Expand Down
17 changes: 1 addition & 16 deletions configs/eval_subjective_mtbench101.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
with read_base():
from .datasets.subjective.multiround.mtbench101_judge import subjective_datasets


from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
from opencompass.partitioners import NaivePartitioner, SizePartitioner
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
Expand All @@ -14,10 +13,6 @@
from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import MTBench101Summarizer





# ---------------------------------------------------------------------------------------------------------

api_meta_template = dict(
Expand All @@ -28,16 +23,6 @@
]
)

# api_meta_template = dict(
# round=[
# dict(role='system', api_role='SYSTEM'),
# dict(role='user', api_role='HUMAN'),
# dict(role='assistant', api_role='BOT', generate=True),
# ]
# )



# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
Expand Down Expand Up @@ -85,7 +70,7 @@
judge_models = [dict(
abbr='GPT4-Turbo',
type=OpenAI,
path='',
path='gpt-4-1106-preview', # To compare with the official leaderboard, please use gpt4-0613
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=16,
Expand Down

0 comments on commit 1ac4513

Please sign in to comment.