Skip to content

Commit

Permalink
Merge branch 'main' into improve-proxy
Browse files Browse the repository at this point in the history 
Conflicts:
	lmdeploy/cli/serve.py
	lmdeploy/serve/proxy/proxy.py
  • Loading branch information
@AllentDan
AllentDan committed Feb 6, 2025
2 parents 6a29e74 + b705fef commit 81440d5
Show file tree
Hide file tree
Showing 360 changed files with 7,757 additions and 14,026 deletions.
64 changes: 18 additions & 46 deletions .github/scripts/action_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,7 @@ def run_cmd(cmd_lines: List[str], log_path: str, cwd: str = None):
with open(log_path, 'w', encoding='utf-8') as file_handler:
file_handler.write(f'Command:\n{cmd_for_log}\n')
file_handler.flush()
process_res = subprocess.Popen(cmd_for_run,
shell=True,
cwd=cwd,
stdout=file_handler,
stderr=file_handler)
process_res = subprocess.Popen(cmd_for_run, shell=True, cwd=cwd, stdout=file_handler, stderr=file_handler)
process_res.wait()
return_code = process_res.returncode

Expand Down Expand Up @@ -79,11 +75,7 @@ def add_summary(csv_path: str):
_append_summary('\n')


def evaluate(models: List[str],
datasets: List[str],
workspace: str,
evaluate_type: str,
is_smoke: bool = False):
def evaluate(models: List[str], datasets: List[str], workspace: str, evaluate_type: str, is_smoke: bool = False):
"""Evaluate models from lmdeploy using opencompass.
Args:
Expand All @@ -103,10 +95,8 @@ def evaluate(models: List[str],

opencompass_dir = os.path.abspath(os.environ['OPENCOMPASS_DIR'])
lmdeploy_dir = os.path.abspath(os.environ['LMDEPLOY_DIR'])
config_path = os.path.join(
lmdeploy_dir, f'.github/scripts/eval_{evaluate_type}_config.py')
config_path_new = os.path.join(opencompass_dir, 'configs',
'eval_lmdeploy.py')
config_path = os.path.join(lmdeploy_dir, f'.github/scripts/eval_{evaluate_type}_config.py')
config_path_new = os.path.join(opencompass_dir, 'configs', 'eval_lmdeploy.py')
if os.path.exists(config_path_new):
os.remove(config_path_new)
shutil.copy(config_path, config_path_new)
Expand Down Expand Up @@ -164,9 +154,7 @@ def evaluate(models: List[str],
row = [_.strip() for _ in row]
if row[-1] != '-':
model_results[row[0]] = row[-1]
crows_pairs_json = glob.glob(os.path.join(
work_dir, '*/results/*/crows_pairs.json'),
recursive=True)
crows_pairs_json = glob.glob(os.path.join(work_dir, '*/results/*/crows_pairs.json'), recursive=True)
if len(crows_pairs_json) == 1:
with open(crows_pairs_json[0], 'r') as f:
acc = json.load(f)['accuracy']
Expand All @@ -175,13 +163,11 @@ def evaluate(models: List[str],
logging.info(f'\n{model}\n{model_results}')
dataset_names = list(model_results.keys())

row = ','.join([model, str(task_duration_seconds)] +
[model_results[_] for _ in dataset_names])
row = ','.join([model, str(task_duration_seconds)] + [model_results[_] for _ in dataset_names])

if not os.path.exists(output_csv):
with open(output_csv, 'w') as f:
header = ','.join(['Model', 'task_duration_secs'] +
dataset_names)
header = ','.join(['Model', 'task_duration_secs'] + dataset_names)
f.write(header + '\n')
f.write(row + '\n')
else:
Expand Down Expand Up @@ -213,30 +199,22 @@ def generate_benchmark_report(report_path: str):
_append_summary('## Benchmark Results Start')
subfolders = [f.path for f in os.scandir(report_path) if f.is_dir()]
for dir_path in subfolders:
second_subfolders = [
f.path for f in sorted(os.scandir(dir_path), key=lambda x: x.name)
if f.is_dir()
]
second_subfolders = [f.path for f in sorted(os.scandir(dir_path), key=lambda x: x.name) if f.is_dir()]
for sec_dir_path in second_subfolders:
model = sec_dir_path.replace(report_path + '/', '')
print('-' * 25, model, '-' * 25)
_append_summary('-' * 25 + model + '-' * 25 + '\n')

benchmark_subfolders = [
f.path
for f in sorted(os.scandir(sec_dir_path), key=lambda x: x.name)
if f.is_dir()
f.path for f in sorted(os.scandir(sec_dir_path), key=lambda x: x.name) if f.is_dir()
]
for backend_subfolder in benchmark_subfolders:
benchmark_type = backend_subfolder.replace(
sec_dir_path + '/', '')
benchmark_type = backend_subfolder.replace(sec_dir_path + '/', '')
print('*' * 10, benchmark_type, '*' * 10)
_append_summary('-' * 10 + benchmark_type + '-' * 10 + '\n')
merged_csv_path = os.path.join(backend_subfolder,
'summary.csv')
merged_csv_path = os.path.join(backend_subfolder, 'summary.csv')
csv_files = glob.glob(os.path.join(backend_subfolder, '*.csv'))
average_csv_path = os.path.join(backend_subfolder,
'average.csv')
average_csv_path = os.path.join(backend_subfolder, 'average.csv')
if merged_csv_path in csv_files:
csv_files.remove(merged_csv_path)
if average_csv_path in csv_files:
Expand All @@ -246,21 +224,18 @@ def generate_benchmark_report(report_path: str):
if len(csv_files) > 0:
for f in csv_files:
df = pd.read_csv(f)
merged_df = pd.concat([merged_df, df],
ignore_index=True)
merged_df = pd.concat([merged_df, df], ignore_index=True)

merged_df = merged_df.sort_values(by=merged_df.columns[0])

grouped_df = merged_df.groupby(merged_df.columns[0])
if 'generation' not in backend_subfolder:
average_values = grouped_df.pipe(
(lambda group: {
'mean': group.mean().round(decimals=3)
}))['mean']
average_values = grouped_df.pipe((lambda group: {
'mean': group.mean().round(decimals=3)
}))['mean']
average_values.to_csv(average_csv_path, index=True)
avg_df = pd.read_csv(average_csv_path)
merged_df = pd.concat([merged_df, avg_df],
ignore_index=True)
merged_df = pd.concat([merged_df, avg_df], ignore_index=True)
add_summary(average_csv_path)
merged_df.to_csv(merged_csv_path, index=False)
if 'generation' in backend_subfolder:
Expand All @@ -287,10 +262,7 @@ def generate_csv_from_profile_result(file_path: str, out_path: str):
import csv
with open(out_path, 'w', newline='') as f:
writer = csv.writer(f)
writer.writerow([
'request_rate', 'completed', 'RPM', 'median_ttft_ms',
'output_throughput'
])
writer.writerow(['request_rate', 'completed', 'RPM', 'median_ttft_ms', 'output_throughput'])
writer.writerows(data_csv)


Expand Down
10 changes: 2 additions & 8 deletions .github/scripts/doc_link_checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,8 @@

def make_parser():
parser = argparse.ArgumentParser('Doc link checker')
parser.add_argument('--http',
default=False,
type=bool,
help='check http or not ')
parser.add_argument('--target',
default='./docs',
type=str,
help='the directory or file to check')
parser.add_argument('--http', default=False, type=bool, help='check http or not ')
parser.add_argument('--target', default='./docs', type=str, help='the directory or file to check')
return parser


Expand Down
108 changes: 36 additions & 72 deletions .github/scripts/eval_base_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,102 +4,67 @@

with read_base():
# choose a list of datasets
from opencompass.configs.datasets.ARC_c.ARC_c_few_shot_ppl import \
ARC_c_datasets # noqa: F401, E501
from opencompass.configs.datasets.bbh.bbh_gen_98fba6 import \
bbh_datasets # noqa: F401, E501
from opencompass.configs.datasets.ceval.ceval_ppl import \
ceval_datasets # noqa: F401, E501
from opencompass.configs.datasets.cmmlu.cmmlu_ppl_041cbf import \
cmmlu_datasets # noqa: F401, E501
from opencompass.configs.datasets.crowspairs.crowspairs_ppl import \
crowspairs_datasets # noqa: F401, E501
from opencompass.configs.datasets.drop.drop_gen_a2697c import \
drop_datasets # noqa: F401, E501
from opencompass.configs.datasets.ARC_c.ARC_c_few_shot_ppl import ARC_c_datasets # noqa: F401, E501
from opencompass.configs.datasets.bbh.bbh_gen_98fba6 import bbh_datasets # noqa: F401, E501
from opencompass.configs.datasets.ceval.ceval_ppl import ceval_datasets # noqa: F401, E501
from opencompass.configs.datasets.cmmlu.cmmlu_ppl_041cbf import cmmlu_datasets # noqa: F401, E501
from opencompass.configs.datasets.crowspairs.crowspairs_ppl import crowspairs_datasets # noqa: F401, E501
from opencompass.configs.datasets.drop.drop_gen_a2697c import drop_datasets # noqa: F401, E501
# Corebench v1.7
from opencompass.configs.datasets.GaokaoBench.GaokaoBench_no_subjective_gen_d21e37 import \
GaokaoBench_datasets # noqa: F401, E501
from opencompass.configs.datasets.gpqa.gpqa_few_shot_ppl_4b5a83 import \
gpqa_datasets # noqa: F401, E501
from opencompass.configs.datasets.gsm8k.gsm8k_gen_17d0dc import \
gsm8k_datasets # noqa: F401, E501
from opencompass.configs.datasets.gpqa.gpqa_few_shot_ppl_4b5a83 import gpqa_datasets # noqa: F401, E501
from opencompass.configs.datasets.gsm8k.gsm8k_gen_17d0dc import gsm8k_datasets # noqa: F401, E501
from opencompass.configs.datasets.hellaswag.hellaswag_10shot_ppl_59c85e import \
hellaswag_datasets # noqa: F401, E501
from opencompass.configs.datasets.humaneval.internal_humaneval_gen_ce6b06 import \
humaneval_datasets as humaneval_v2_datasets # noqa: F401, E501
from opencompass.configs.datasets.humaneval.internal_humaneval_gen_d2537e import \
humaneval_datasets # noqa: F401, E501
from opencompass.configs.datasets.math.math_4shot_base_gen_43d5b6 import \
math_datasets # noqa: F401, E501
from opencompass.configs.datasets.math.math_4shot_base_gen_43d5b6 import math_datasets # noqa: F401, E501
from opencompass.configs.datasets.MathBench.mathbench_2024_few_shot_mixed_4a3fd4 import \
mathbench_datasets # noqa: F401, E501
from opencompass.configs.datasets.mbpp.sanitized_mbpp_gen_742f0c import \
sanitized_mbpp_datasets # noqa: F401, E501
from opencompass.configs.datasets.mmlu.mmlu_ppl_ac766d import \
mmlu_datasets # noqa: F401, E501
from opencompass.configs.datasets.mmlu_pro.mmlu_pro_few_shot_gen_bfaf90 import \
mmlu_pro_datasets # noqa: F401, E501
from opencompass.configs.datasets.nq.nq_open_1shot_gen_20a989 import \
nq_datasets # noqa: F401, E501
from opencompass.configs.datasets.race.race_few_shot_ppl import \
race_datasets # noqa: F401, E501
from opencompass.configs.datasets.mbpp.sanitized_mbpp_gen_742f0c import sanitized_mbpp_datasets # noqa: F401, E501
from opencompass.configs.datasets.mmlu.mmlu_ppl_ac766d import mmlu_datasets # noqa: F401, E501
from opencompass.configs.datasets.mmlu_pro.mmlu_pro_few_shot_gen_bfaf90 import mmlu_pro_datasets # noqa: F401, E501
from opencompass.configs.datasets.nq.nq_open_1shot_gen_20a989 import nq_datasets # noqa: F401, E501
from opencompass.configs.datasets.race.race_few_shot_ppl import race_datasets # noqa: F401, E501
from opencompass.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_few_shot_ppl import \
BoolQ_datasets # noqa: F401, E501
from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import \
TheoremQA_datasets # noqa: F401, E501
from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import TheoremQA_datasets # noqa: F401, E501
from opencompass.configs.datasets.triviaqa.triviaqa_wiki_1shot_gen_20a989 import \
triviaqa_datasets # noqa: F401, E501
from opencompass.configs.datasets.wikibench.wikibench_few_shot_ppl_c23d79 import \
wikibench_datasets # noqa: F401, E501
from opencompass.configs.datasets.winogrande.winogrande_5shot_ll_252f01 import \
winogrande_datasets # noqa: F401, E501
from opencompass.configs.models.baichuan.hf_baichuan_7b import \
models as hf_baichuan_7b # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma_7b import \
models as hf_gemma_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b import \
models as hf_internlm2_5_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_7b import \
models as hf_internlm2_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_20b import \
models as hf_internlm2_20b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm_7b import \
models as hf_internlm_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm_20b import \
models as hf_internlm_20b # noqa: F401, E501
from opencompass.configs.models.baichuan.hf_baichuan_7b import models as hf_baichuan_7b # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma_7b import models as hf_gemma_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b import models as hf_internlm2_5_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_7b import models as hf_internlm2_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_20b import models as hf_internlm2_20b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm_7b import models as hf_internlm_7b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm_20b import models as hf_internlm_20b # noqa: F401, E501
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b import \
models as lmdeploy_internlm2_5_7b # noqa: F401, E501
from opencompass.configs.models.hf_llama.hf_llama2_7b import \
models as hf_llama2_7b # noqa: F401, E501
from opencompass.configs.models.hf_llama.hf_llama3_8b import \
models as hf_llama3_8b # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mistral_7b_v0_1 import \
models as hf_mistral_7b_v0_1 # noqa: F401, E501
from opencompass.configs.models.hf_llama.hf_llama2_7b import models as hf_llama2_7b # noqa: F401, E501
from opencompass.configs.models.hf_llama.hf_llama3_8b import models as hf_llama3_8b # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mistral_7b_v0_1 import models as hf_mistral_7b_v0_1 # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mixtral_8x7b_v0_1 import \
models as hf_mixtral_8x7b_v0_1 # noqa: F401, E501
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b import \
models as lmdeploy_qwen2_5_7b # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen1_5_7b import \
models as hf_qwen1_5_7b # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen2_7b import \
models as hf_qwen2_7b # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen_7b import \
models as hf_qwen_7b # noqa: F401, E501
from opencompass.configs.models.qwen.lmdeploy_qwen1_5_7b import \
models as lmdeploy_qwen1_5_7b # noqa: F401, E501
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b import \
models as lmdeploy_qwen2_7b # noqa: F401, E501
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b import models as lmdeploy_qwen2_5_7b # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen1_5_7b import models as hf_qwen1_5_7b # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen2_7b import models as hf_qwen2_7b # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen_7b import models as hf_qwen_7b # noqa: F401, E501
from opencompass.configs.models.qwen.lmdeploy_qwen1_5_7b import models as lmdeploy_qwen1_5_7b # noqa: F401, E501
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b import models as lmdeploy_qwen2_7b # noqa: F401, E501
# Summary Groups
from opencompass.configs.summarizers.groups.cmmlu import \
cmmlu_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.GaokaoBench import \
GaokaoBench_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.cmmlu import cmmlu_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.GaokaoBench import GaokaoBench_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mathbench_v1_2024 import \
mathbench_2024_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mmlu import \
mmlu_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mmlu_pro import \
mmlu_pro_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mmlu import mmlu_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mmlu_pro import mmlu_pro_summary_groups # noqa: F401, E501

# read models
race_datasets = [race_datasets[1]]
Expand Down Expand Up @@ -169,8 +134,7 @@
'mmlu_pro_psychology',
'mmlu_pro_other',
],
summary_groups=sum(
[v for k, v in locals().items() if k.endswith('_summary_groups')], []),
summary_groups=sum([v for k, v in locals().items() if k.endswith('_summary_groups')], []),
)

turbomind_qwen1_5_7b = deepcopy(*lmdeploy_qwen1_5_7b)
Expand Down
Loading

0 comments on commit 81440d5

Please sign in to comment.