A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 13:00 UTC
Add HumanEval and GSM8K benchmarks to datasets (#42648)
Signed-off-by: southfreebird <yvorott@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Sergei Skvortsov committed
May 15, 2026 at 21:01 UTC
9a7a273dfe6a89bbe00639fe99b0d61095fbc40a
2 files changed
+155
docs/benchmarking/cli.md
+22
@@ -34,6 +34,8 @@ th {
34
| HuggingFace-AIMO | ✅ | ✅ | `AI-MO/aimo-validation-aime`, `AI-MO/NuminaMath-1.5`, `AI-MO/NuminaMath-CoT` |
35
| HuggingFace-Other | ✅ | ✅ | `lmms-lab/LLaVA-OneVision-Data`, `Aeala/ShareGPT_Vicuna_unfiltered` |
36
| HuggingFace-MTBench | ✅ | ✅ | `philschmid/mt-bench` |
37
+| HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` |
38
+| HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` |
39
| HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |
40
| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` |
41
| Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |
@@ -443,6 +445,26 @@ vllm bench serve \
445
--num-prompts 80
446
```
447
448
+`openai/openai_humaneval`:
449
+
450
+``` bash
451
+vllm bench serve \
452
+ --model NousResearch/Hermes-3-Llama-3.1-8B \
453
+ --dataset-name hf \
454
+ --dataset-path openai/openai_humaneval \
455
+ --num-prompts 80
456
+```
457
+
458
+`openai/gsm8k`:
459
+
460
+``` bash
461
+vllm bench serve \
462
+ --model NousResearch/Hermes-3-Llama-3.1-8B \
463
+ --dataset-name hf \
464
+ --dataset-path openai/gsm8k \
465
+ --num-prompts 80
466
+```
467
+
468
`vdaita/edit_5k_char` or `vdaita/edit_10k_char`:
469
470
``` bash
vllm/benchmarks/datasets/datasets.py
+133
@@ -1940,6 +1940,19 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1940
):
1941
dataset_class = MTBenchDataset
1942
args.hf_split = args.hf_split if args.hf_split else "train"
1943
+ elif (
1944
+ args.dataset_path in HumanEvalDataset.SUPPORTED_DATASET_PATHS
1945
+ or args.hf_name in HumanEvalDataset.SUPPORTED_DATASET_PATHS
1946
+ ):
1947
+ dataset_class = HumanEvalDataset
1948
+ args.hf_split = args.hf_split if args.hf_split else "test"
1949
+ elif (
1950
+ args.dataset_path in GSM8KDataset.SUPPORTED_DATASET_PATHS
1951
+ or args.hf_name in GSM8KDataset.SUPPORTED_DATASET_PATHS
1952
+ ):
1953
+ dataset_class = GSM8KDataset
1954
+ args.hf_subset = args.hf_subset if args.hf_subset else "main"
1955
+ args.hf_split = args.hf_split if args.hf_split else "test"
1956
elif (
1957
args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
1958
or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
@@ -3122,6 +3135,126 @@ class MTBenchDataset(HuggingFaceDataset):
3135
return sampled_requests
3136
3137
3138
+# -----------------------------------------------------------------------------
3139
+# HumanEval Dataset Implementation
3140
+# -----------------------------------------------------------------------------
3141
+
3142
+
3143
+class HumanEvalDataset(HuggingFaceDataset):
3144
+ """
3145
+ HumanEvalDataset Dataset.
3146
+ https://huggingface.co/datasets/openai/openai_humaneval
3147
+
3148
+ We create a single turn dataset for HumanEval.
3149
+ """
3150
+
3151
+ DEFAULT_OUTPUT_LEN = 256
3152
+ SUPPORTED_DATASET_PATHS = {
3153
+ "openai/openai_humaneval",
3154
+ }
3155
+
3156
+ def sample(
3157
+ self,
3158
+ tokenizer: TokenizerLike,
3159
+ num_requests: int,
3160
+ request_id_prefix: str = "",
3161
+ no_oversample: bool = False,
3162
+ output_len: int | None = None,
3163
+ enable_multimodal_chat: bool = False,
3164
+ skip_chat_template: bool = False,
3165
+ **kwargs,
3166
+ ) -> list[SampleRequest]:
3167
+ output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
3168
+ sampled_requests = []
3169
+
3170
+ for i, item in enumerate(self.data):
3171
+ if len(sampled_requests) >= num_requests:
3172
+ break
3173
+ prompt = item["prompt"]
3174
+
3175
+ # apply template
3176
+ if not skip_chat_template:
3177
+ prompt = tokenizer.apply_chat_template(
3178
+ [{"role": "user", "content": prompt}],
3179
+ add_generation_prompt=True,
3180
+ tokenize=False,
3181
+ )
3182
+
3183
+ prompt_len = len(tokenizer(prompt).input_ids)
3184
+ sampled_requests.append(
3185
+ SampleRequest(
3186
+ prompt=prompt,
3187
+ prompt_len=prompt_len,
3188
+ expected_output_len=output_len,
3189
+ request_id=request_id_prefix + str(i),
3190
+ )
3191
+ )
3192
+ self.maybe_oversample_requests(
3193
+ sampled_requests, num_requests, request_id_prefix, no_oversample
3194
+ )
3195
+ return sampled_requests
3196
+
3197
+
3198
+# -----------------------------------------------------------------------------
3199
+# GSM8K Dataset Implementation
3200
+# -----------------------------------------------------------------------------
3201
+
3202
+
3203
+class GSM8KDataset(HuggingFaceDataset):
3204
+ """
3205
+ GSM8K Dataset.
3206
+ https://huggingface.co/datasets/openai/gsm8k
3207
+
3208
+ We create a single turn dataset for GSM8K.
3209
+ """
3210
+
3211
+ DEFAULT_OUTPUT_LEN = 256
3212
+ SUPPORTED_DATASET_PATHS = {
3213
+ "openai/gsm8k",
3214
+ }
3215
+
3216
+ def sample(
3217
+ self,
3218
+ tokenizer: TokenizerLike,
3219
+ num_requests: int,
3220
+ request_id_prefix: str = "",
3221
+ no_oversample: bool = False,
3222
+ output_len: int | None = None,
3223
+ enable_multimodal_chat: bool = False,
3224
+ skip_chat_template: bool = False,
3225
+ **kwargs,
3226
+ ) -> list[SampleRequest]:
3227
+ output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
3228
+ sampled_requests = []
3229
+
3230
+ for i, item in enumerate(self.data):
3231
+ if len(sampled_requests) >= num_requests:
3232
+ break
3233
+ prompt = item["question"]
3234
+
3235
+ # apply template
3236
+ if not skip_chat_template:
3237
+ prompt = tokenizer.apply_chat_template(
3238
+ [{"role": "user", "content": prompt}],
3239
+ add_generation_prompt=True,
3240
+ tokenize=False,
3241
+ )
3242
+
3243
+ prompt_len = len(tokenizer(prompt).input_ids)
3244
+ sampled_requests.append(
3245
+ SampleRequest(
3246
+ prompt=prompt,
3247
+ prompt_len=prompt_len,
3248
+ expected_output_len=output_len,
3249
+ request_id=request_id_prefix + str(i),
3250
+ )
3251
+ )
3252
+ self.maybe_oversample_requests(
3253
+ sampled_requests, num_requests, request_id_prefix, no_oversample
3254
+ )
3255
+ return sampled_requests
3256
+
3257
+
3258
# -----------------------------------------------------------------------------
3259
# Blazedit Dataset Implementation
3260
# -----------------------------------------------------------------------------