obscura/vllm
2
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-08-20 20:50:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add HumanEval and GSM8K benchmarks to datasets (#42648)

Browse Source 
Signed-off-by: southfreebird <[email protected]>
Co-authored-by: Michael Goin <[email protected]>
This commit is contained in:
Sergei Skvortsov
2026-05-15 13:01:21 -07:00
committed by GitHub
co-authored by Michael Goin
parent b2c58ee942
commit 9a7a273dfe
2 changed files with 155 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/benchmarking/cli.md
+22
View File
@@ -34,6 +34,8 @@ th {
| HuggingFace-AIMO | ✅ | ✅ | `AI-MO/aimo-validation-aime`, `AI-MO/NuminaMath-1.5`, `AI-MO/NuminaMath-CoT` |
| HuggingFace-Other | ✅ | ✅ | `lmms-lab/LLaVA-OneVision-Data`, `Aeala/ShareGPT_Vicuna_unfiltered` |
| HuggingFace-MTBench | ✅ | ✅ | `philschmid/mt-bench` |
| HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` |
| HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` |
| HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |
| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` |
| Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |
@@ -443,6 +445,26 @@ vllm bench serve \
--num-prompts 80
```
`openai/openai_humaneval`:
``` bash
vllm bench serve \
--model NousResearch/Hermes-3-Llama-3.1-8B \
--dataset-name hf \
--dataset-path openai/openai_humaneval \
--num-prompts 80
```
`openai/gsm8k`:
``` bash
vllm bench serve \
--model NousResearch/Hermes-3-Llama-3.1-8B \
--dataset-name hf \
--dataset-path openai/gsm8k \
--num-prompts 80
```
`vdaita/edit_5k_char` or `vdaita/edit_10k_char`:
``` bash
vllm/benchmarks/datasets/datasets.py
+133
View File
@@ -1940,6 +1940,19 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
):
dataset_class = MTBenchDataset
args.hf_split = args.hf_split if args.hf_split else "train"
elif (
args.dataset_path in HumanEvalDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in HumanEvalDataset.SUPPORTED_DATASET_PATHS
):
dataset_class = HumanEvalDataset
args.hf_split = args.hf_split if args.hf_split else "test"
elif (
args.dataset_path in GSM8KDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in GSM8KDataset.SUPPORTED_DATASET_PATHS
):
dataset_class = GSM8KDataset
args.hf_subset = args.hf_subset if args.hf_subset else "main"
args.hf_split = args.hf_split if args.hf_split else "test"
elif (
args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
@@ -3122,6 +3135,126 @@ class MTBenchDataset(HuggingFaceDataset):
return sampled_requests
# -----------------------------------------------------------------------------
# HumanEval Dataset Implementation
# -----------------------------------------------------------------------------
class HumanEvalDataset(HuggingFaceDataset):
"""
HumanEvalDataset Dataset.
https://huggingface.co/datasets/openai/openai_humaneval
We create a single turn dataset for HumanEval.
"""
DEFAULT_OUTPUT_LEN = 256
SUPPORTED_DATASET_PATHS = {
"openai/openai_humaneval",
}
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
prompt = item["prompt"]
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
prompt_len = len(tokenizer(prompt).input_ids)
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(i),
)
)
self.maybe_oversample_requests(
sampled_requests, num_requests, request_id_prefix, no_oversample
)
return sampled_requests
# -----------------------------------------------------------------------------
# GSM8K Dataset Implementation
# -----------------------------------------------------------------------------
class GSM8KDataset(HuggingFaceDataset):
"""
GSM8K Dataset.
https://huggingface.co/datasets/openai/gsm8k
We create a single turn dataset for GSM8K.
"""
DEFAULT_OUTPUT_LEN = 256
SUPPORTED_DATASET_PATHS = {
"openai/gsm8k",
}
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
prompt = item["question"]
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
prompt_len = len(tokenizer(prompt).input_ids)
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(i),
)
)
self.maybe_oversample_requests(
sampled_requests, num_requests, request_id_prefix, no_oversample
)
return sampled_requests
# -----------------------------------------------------------------------------
# Blazedit Dataset Implementation
# -----------------------------------------------------------------------------