A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Benchmark] Allow oversample request in benchmark dataset (#15170)
Signed-off-by: Jennifer Zhao <ai.jenniferzhao@gmail.com>
Jennifer Zhao committed
Mar 19, 2025 at 21:32 UTC
b88be2216549a7d2281153a8542fca2aea06681c
2 files changed
+139
-59
benchmarks/README.md
+54
-3
@@ -42,7 +42,7 @@ become available.
42
</tr>
43
<tr>
44
<td><strong>HuggingFace</strong></td>
45
- <td style="text-align: center;">✅</td>
45
+ <td style="text-align: center;">🟡</td>
46
<td style="text-align: center;">🟡</td>
47
<td>Specify your dataset path on HuggingFace</td>
48
</tr>
@@ -60,8 +60,8 @@ become available.
60
🚧: to be supported
61
62
🟡: Partial support. Currently, HuggingFaceDataset only supports dataset formats
63
-similar to `lmms-lab/LLaVA-OneVision-Data`. If you need support for other dataset
64
-formats, please consider contributing.
63
+similar to `lmms-lab/LLaVA-OneVision-Data` and `Aeala/ShareGPT_Vicuna_unfiltered`.
64
+If you need support for other dataset formats, please consider contributing.
65
66
**Note**: VisionArena’s `dataset-name` should be set to `hf`
67
@@ -139,6 +139,57 @@ python3 vllm/benchmarks/benchmark_serving.py \
139
--num-prompts "${NUM_PROMPTS}"
140
```
141
142
+### HuggingFaceDataset Examples
143
+
144
+Currently, HuggingFaceDataset only supports dataset formats
145
+similar to `lmms-lab/LLaVA-OneVision-Data` and `Aeala/ShareGPT_Vicuna_unfiltered`. If you need support for other dataset
146
+formats, please consider contributing.
147
+
148
+```bash
149
+# need a model with vision capability here
150
+vllm serve Qwen/Qwen2-VL-7B-Instruct --disable-log-requests
151
+```
152
+
153
+**`lmms-lab/LLaVA-OneVision-Data`**
154
+
155
+```bash
156
+MODEL_NAME="Qwen/Qwen2-VL-7B-Instruct"
157
+NUM_PROMPTS=10
158
+BACKEND="openai-chat"
159
+DATASET_NAME="hf"
160
+DATASET_PATH="lmms-lab/LLaVA-OneVision-Data"
161
+DATASET_SPLIT='train'
162
+DATASET_SUBSET='chart2text(cauldron)'
163
+python3 vllm/benchmarks/benchmark_serving.py \
164
+ --backend "${BACKEND}" \
165
+ --model "${MODEL_NAME}" \
166
+ --endpoint "/v1/chat/completions" \
167
+ --dataset-name "${DATASET_NAME}" \
168
+ --dataset-path "${DATASET_PATH}" \
169
+ --hf-split "${DATASET_SPLIT}" \
170
+ --num-prompts "${NUM_PROMPTS}" \
171
+ --hf-subset "${DATASET_SUBSET}"
172
+```
173
+
174
+**`Aeala/ShareGPT_Vicuna_unfiltered`**
175
+
176
+```bash
177
+MODEL_NAME="Qwen/Qwen2-VL-7B-Instruct"
178
+NUM_PROMPTS=10
179
+BACKEND="openai-chat"
180
+DATASET_NAME="hf"
181
+DATASET_PATH="Aeala/ShareGPT_Vicuna_unfiltered"
182
+DATASET_SPLIT='train'
183
+python3 vllm/benchmarks/benchmark_serving.py \
184
+ --backend "${BACKEND}" \
185
+ --model "${MODEL_NAME}" \
186
+ --endpoint "/v1/chat/completions" \
187
+ --dataset-name "${DATASET_NAME}" \
188
+ --dataset-path "${DATASET_PATH}" \
189
+ --hf-split "${DATASET_SPLIT}" \
190
+ --num-prompts "${NUM_PROMPTS}" \
191
+```
192
+
193
---
194
## Example - Offline Throughput Benchmark
195
benchmarks/benchmark_dataset.py
+85
-56
@@ -17,6 +17,7 @@ SampleRequest instances, similar to the approach used in ShareGPT.
17
import base64
18
import io
19
import json
20
+import logging
21
import random
22
from abc import ABC, abstractmethod
23
from collections.abc import Mapping
@@ -35,6 +36,8 @@ from vllm.lora.utils import get_adapter_absolute_path
36
from vllm.multimodal import MultiModalDataDict
37
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer
38
39
+logger = logging.getLogger(__name__)
40
+
41
# -----------------------------------------------------------------------------
42
# Data Classes
43
# -----------------------------------------------------------------------------
@@ -61,9 +64,6 @@ class SampleRequest:
64
class BenchmarkDataset(ABC):
65
DEFAULT_SEED = 0
66
64
- # num_requests has default 1000 in both the benchmark_serving.py and
65
- # benchmark_throughput.py
66
-
67
def __init__(
68
self,
69
dataset_path: Optional[str] = None,
@@ -90,8 +90,8 @@ class BenchmarkDataset(ABC):
90
mm_content: Optional[MultiModalDataDict] = None) -> list[dict]:
91
"""
92
Transform a prompt and optional multimodal content into a chat format.
93
- This method is used for chat models that expect a specific
94
- conversation format.
93
+ This method is used for chat models that expect a specific conversation
94
+ format.
95
"""
96
content = [{"text": prompt, "type": "text"}]
97
if mm_content is not None:
@@ -101,10 +101,10 @@ class BenchmarkDataset(ABC):
101
def load_data(self) -> None:
102
"""
103
Load data from the dataset path into self.data.
104
-
104
+
105
This method must be overridden by subclasses since the method to load
106
data will vary depending on the dataset format and source.
107
-
107
+
108
Raises:
109
NotImplementedError: If a subclass does not implement this method.
110
"""
@@ -121,18 +121,18 @@ class BenchmarkDataset(ABC):
121
"""
122
Optionally select a random LoRA request and return its associated
123
tokenizer.
124
-
124
+
125
This method is used when LoRA parameters are provided. It randomly
126
selects a LoRA based on max_loras and retrieves a cached tokenizer for
127
that LoRA if available. Otherwise, it returns the base tokenizer.
128
-
128
+
129
Args:
130
tokenizer (PreTrainedTokenizerBase): The base tokenizer to use if no
131
LoRA is selected. max_loras (Optional[int]): The maximum number of
132
LoRAs available. If None, LoRA is not used. lora_path
133
(Optional[str]): Path to the LoRA parameters on disk. If None, LoRA
134
is not used.
135
-
135
+
136
Returns:
137
tuple[Optional[LoRARequest], AnyTokenizer]: A tuple where the first
138
element is a LoRARequest (or None if not applicable) and the second
@@ -160,21 +160,39 @@ class BenchmarkDataset(ABC):
160
num_requests: int) -> list[SampleRequest]:
161
"""
162
Abstract method to generate sample requests from the dataset.
163
-
163
+
164
Subclasses must override this method to implement dataset-specific logic
165
for generating a list of SampleRequest objects.
166
-
166
+
167
Args:
168
tokenizer (PreTrainedTokenizerBase): The tokenizer to be used
169
for processing the dataset's text.
170
num_requests (int): The number of sample requests to generate.
171
-
171
+
172
Returns:
173
list[SampleRequest]: A list of sample requests generated from the
174
dataset.
175
"""
176
raise NotImplementedError("sample must be implemented in subclasses.")
177
178
+ def maybe_oversample_requests(self, requests: list[SampleRequest],
179
+ num_requests: int) -> None:
180
+ """
181
+ Oversamples the list of requests if its size is less than the desired
182
+ number.
183
+
184
+ Args:
185
+ requests (List[SampleRequest]): The current list of sampled
186
+ requests. num_requests (int): The target number of requests.
187
+ """
188
+ if len(requests) < num_requests:
189
+ random.seed(self.random_seed)
190
+ additional = random.choices(requests,
191
+ k=num_requests - len(requests))
192
+ requests.extend(additional)
193
+ logger.info("Oversampled requests to reach %d total samples.",
194
+ num_requests)
195
+
196
197
# -----------------------------------------------------------------------------
198
# Utility Functions and Global Caches
@@ -276,15 +294,16 @@ class RandomDataset(BenchmarkDataset):
294
) -> None:
295
super().__init__(**kwargs)
296
279
- def sample(self,
280
- tokenizer: PreTrainedTokenizerBase,
281
- num_requests: int,
282
- prefix_len: int = DEFAULT_PREFIX_LEN,
283
- range_ratio: float = DEFAULT_RANGE_RATIO,
284
- input_len: int = DEFAULT_INPUT_LEN,
285
- output_len: int = DEFAULT_OUTPUT_LEN,
286
- **kwargs) -> list[SampleRequest]:
287
-
297
+ def sample(
298
+ self,
299
+ tokenizer: PreTrainedTokenizerBase,
300
+ num_requests: int,
301
+ prefix_len: int = DEFAULT_PREFIX_LEN,
302
+ range_ratio: float = DEFAULT_RANGE_RATIO,
303
+ input_len: int = DEFAULT_INPUT_LEN,
304
+ output_len: int = DEFAULT_OUTPUT_LEN,
305
+ **kwargs,
306
+ ) -> list[SampleRequest]:
307
vocab_size = tokenizer.vocab_size
308
309
prefix_token_ids = (np.random.randint(
@@ -346,20 +365,24 @@ class ShareGPTDataset(BenchmarkDataset):
365
random.seed(self.random_seed)
366
random.shuffle(self.data)
367
349
- def sample(self,
350
- tokenizer: PreTrainedTokenizerBase,
351
- num_requests: int,
352
- lora_path: Optional[str] = None,
353
- max_loras: Optional[int] = None,
354
- output_len: Optional[int] = None,
355
- enable_multimodal_chat: bool = False,
356
- **kwargs) -> list:
368
+ def sample(
369
+ self,
370
+ tokenizer: PreTrainedTokenizerBase,
371
+ num_requests: int,
372
+ lora_path: Optional[str] = None,
373
+ max_loras: Optional[int] = None,
374
+ output_len: Optional[int] = None,
375
+ enable_multimodal_chat: bool = False,
376
+ **kwargs,
377
+ ) -> list:
378
samples: list = []
379
for entry in self.data:
380
if len(samples) >= num_requests:
381
break
361
- prompt, completion = entry["conversations"][0]["value"],\
362
- entry["conversations"][1]["value"]
382
+ prompt, completion = (
383
+ entry["conversations"][0]["value"],
384
+ entry["conversations"][1]["value"],
385
+ )
386
387
lora_request, tokenizer = self.get_random_lora_request(
388
tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path)
@@ -383,6 +406,7 @@ class ShareGPTDataset(BenchmarkDataset):
406
expected_output_len=new_output_len,
407
lora_request=lora_request,
408
))
409
+ self.maybe_oversample_requests(samples, num_requests)
410
return samples
411
412
@@ -415,19 +439,20 @@ class SonnetDataset(BenchmarkDataset):
439
with open(self.dataset_path, encoding="utf-8") as f:
440
self.data = f.readlines()
441
418
- def sample(self,
419
- tokenizer,
420
- num_requests: int,
421
- prefix_len: int = DEFAULT_PREFIX_LEN,
422
- input_len: int = DEFAULT_INPUT_LEN,
423
- output_len: int = DEFAULT_OUTPUT_LEN,
424
- return_prompt_formatted: bool = False,
425
- **kwargs) -> list:
442
+ def sample(
443
+ self,
444
+ tokenizer,
445
+ num_requests: int,
446
+ prefix_len: int = DEFAULT_PREFIX_LEN,
447
+ input_len: int = DEFAULT_INPUT_LEN,
448
+ output_len: int = DEFAULT_OUTPUT_LEN,
449
+ return_prompt_formatted: bool = False,
450
+ **kwargs,
451
+ ) -> list:
452
# Calculate average token length for a poem line.
453
tokenized_lines = [tokenizer(line).input_ids for line in self.data]
454
avg_len = sum(len(tokens)
429
- for tokens in \
430
- tokenized_lines) / len(tokenized_lines)
455
+ for tokens in tokenized_lines) / len(tokenized_lines)
456
457
# Build the base prompt.
458
base_prompt = "Pick as many lines as you can from these poem lines:\n"
@@ -506,12 +531,14 @@ class BurstGPTDataset(BenchmarkDataset):
531
# Convert the dataframe to a list of lists.
532
return data.values.tolist()
533
509
- def sample(self,
510
- tokenizer: PreTrainedTokenizerBase,
511
- num_requests: int,
512
- max_loras: Optional[int] = None,
513
- lora_path: Optional[str] = None,
514
- **kwargs) -> list[SampleRequest]:
534
+ def sample(
535
+ self,
536
+ tokenizer: PreTrainedTokenizerBase,
537
+ num_requests: int,
538
+ max_loras: Optional[int] = None,
539
+ lora_path: Optional[str] = None,
540
+ **kwargs,
541
+ ) -> list[SampleRequest]:
542
samples = []
543
data = self._sample_loaded_data(num_requests=num_requests)
544
for i in range(num_requests):
@@ -544,7 +571,6 @@ class HuggingFaceDataset(BenchmarkDataset):
571
Dataset class for processing a HuggingFace dataset with conversation data
572
and optional images.
573
"""
547
- DEFAULT_NUM_REQUESTS = 1000
574
575
def __init__(
576
self,
@@ -618,6 +644,7 @@ class HuggingFaceDataset(BenchmarkDataset):
644
expected_output_len=output_len,
645
multi_modal_data=mm_content,
646
))
647
+ self.maybe_oversample_requests(sampled_requests, num_requests)
648
return sampled_requests
649
650
@@ -632,7 +659,6 @@ class VisionArenaDataset(HuggingFaceDataset):
659
"""
660
661
DEFAULT_OUTPUT_LEN = 128
635
- DEFAULT_NUM_REQUESTS = 1000
662
VISION_ARENA_DATASET_PATH = "lmarena-ai/vision-arena-bench-v0.1"
663
664
def __init__(
@@ -657,12 +683,14 @@ class VisionArenaDataset(HuggingFaceDataset):
683
)
684
self.data = dataset.shuffle(seed=self.random_seed)
685
660
- def sample(self,
661
- tokenizer: PreTrainedTokenizerBase,
662
- num_requests: int,
663
- output_len: Optional[int] = None,
664
- enable_multimodal_chat: bool = False,
665
- **kwargs) -> list:
686
+ def sample(
687
+ self,
688
+ tokenizer: PreTrainedTokenizerBase,
689
+ num_requests: int,
690
+ output_len: Optional[int] = None,
691
+ enable_multimodal_chat: bool = False,
692
+ **kwargs,
693
+ ) -> list:
694
output_len = (output_len
695
if output_len is not None else self.DEFAULT_OUTPUT_LEN)
696
sampled_requests = []
@@ -685,4 +713,5 @@ class VisionArenaDataset(HuggingFaceDataset):
713
expected_output_len=output_len,
714
multi_modal_data=mm_content,
715
))
716
+ self.maybe_oversample_requests(sampled_requests, num_requests)
717
return sampled_requests