A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Benchmark][Doc] Update throughput benchmark and README (#15998)
Signed-off-by: StevenShi-23 <shi.ziji.sm@gmail.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Ziji Shi (Steven) committed
Apr 4, 2025 at 09:39 UTC
95862f7b4de2fbe99529a4832e2d8424bc9ee247
2 files changed
+44
-11
benchmarks/README.md
+29
@@ -51,6 +51,12 @@ become available.
51
<td style="text-align: center;">✅</td>
52
<td style="text-align: center;">✅</td>
53
<td><code>likaixin/InstructCoder</code></td>
54
+ </tr>
55
+ <tr>
56
+ <td><strong>HuggingFace-AIMO</strong></td>
57
+ <td style="text-align: center;">✅</td>
58
+ <td style="text-align: center;">✅</td>
59
+ <td><code>AI-MO/aimo-validation-aime</code> , <code>AI-MO/NuminaMath-1.5</code>, <code>AI-MO/NuminaMath-CoT</code></td>
60
</tr>
61
<tr>
62
<td><strong>HuggingFace-Other</strong></td>
@@ -187,6 +193,17 @@ python3 vllm/benchmarks/benchmark_serving.py \
193
--num-prompts 10
194
```
195
196
+**`AI-MO/aimo-validation-aime`**
197
+
198
+``` bash
199
+python3 vllm/benchmarks/benchmark_serving.py \
200
+ --model Qwen/QwQ-32B \
201
+ --dataset-name hf \
202
+ --dataset-path AI-MO/aimo-validation-aime \
203
+ --num-prompts 10 \
204
+ --seed 42
205
+```
206
+
207
---
208
## Example - Offline Throughput Benchmark
209
@@ -278,6 +295,18 @@ python3 vllm/benchmarks/benchmark_throughput.py \
295
--num-prompts 10
296
```
297
298
+**`AI-MO/aimo-validation-aime`**
299
+
300
+```bash
301
+python3 benchmarks/benchmark_throughput.py \
302
+ --model Qwen/QwQ-32B \
303
+ --backend vllm \
304
+ --dataset-name hf \
305
+ --dataset-path AI-MO/aimo-validation-aime \
306
+ --hf-split train \
307
+ --num-prompts 10
308
+```
309
+
310
### Benchmark with LoRA Adapters
311
312
``` bash
benchmarks/benchmark_throughput.py
+15
-11
@@ -11,10 +11,10 @@ from typing import Any, Optional, Union
11
12
import torch
13
import uvloop
14
-from benchmark_dataset import (BurstGPTDataset, ConversationDataset,
15
- InstructCoderDataset, RandomDataset,
16
- SampleRequest, ShareGPTDataset, SonnetDataset,
17
- VisionArenaDataset)
14
+from benchmark_dataset import (AIMODataset, BurstGPTDataset,
15
+ ConversationDataset, InstructCoderDataset,
16
+ RandomDataset, SampleRequest, ShareGPTDataset,
17
+ SonnetDataset, VisionArenaDataset)
18
from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json
19
from tqdm import tqdm
20
from transformers import (AutoModelForCausalLM, AutoTokenizer,
@@ -332,7 +332,10 @@ def get_requests(args, tokenizer):
332
common_kwargs['dataset_subset'] = args.hf_subset
333
common_kwargs['dataset_split'] = args.hf_split
334
sample_kwargs["enable_multimodal_chat"] = True
335
-
335
+ elif args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS:
336
+ dataset_cls = AIMODataset
337
+ common_kwargs['dataset_subset'] = None
338
+ common_kwargs['dataset_split'] = "train"
339
else:
340
raise ValueError(f"Unknown dataset name: {args.dataset_name}")
341
# Remove None values
@@ -467,12 +470,13 @@ def validate_args(args):
470
since --dataset-name is not 'hf'.",
471
stacklevel=2)
472
elif args.dataset_name == "hf":
470
- if args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS:
471
- assert args.backend == "vllm-chat", "VisionArenaDataset needs to use vllm-chat as the backend." #noqa: E501
472
- elif args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS:
473
- assert args.backend == "vllm", "InstructCoder dataset needs to use vllm as the backend." #noqa: E501
474
- elif args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS:
475
- assert args.backend == "vllm-chat", "ConversationDataset needs to use vllm-chat as the backend." #noqa: E501
473
+ if args.dataset_path in (
474
+ VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
475
+ | ConversationDataset.SUPPORTED_DATASET_PATHS):
476
+ assert args.backend == "vllm-chat", f"{args.dataset_path} needs to use vllm-chat as the backend." #noqa: E501
477
+ elif args.dataset_path in (InstructCoderDataset.SUPPORTED_DATASET_PATHS
478
+ | AIMODataset.SUPPORTED_DATASET_PATHS):
479
+ assert args.backend == "vllm", f"{args.dataset_path} needs to use vllm as the backend." #noqa: E501
480
else:
481
raise ValueError(
482
f"{args.dataset_path} is not supported by hf dataset.")