A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 04:15 UTC
[Benchmark] Add num-warmup to vllm bench throughput (#43245)
Signed-off-by: Yifan Zong <yzong@redhat.com>
yzong-rh committed
May 21, 2026 at 01:13 UTC
905b97adfaf7b08f3cc95b328579e5336ed6d3b6
1 file changed
+201
-55
vllm/benchmarks/throughput.py
+201
-55
@@ -50,22 +50,59 @@ def run_vllm(
50
engine_args: EngineArgs,
51
do_profile: bool,
52
disable_detokenize: bool = False,
53
+ warmup_requests: list[SampleRequest] | None = None,
54
prequeue_requests: bool = False,
55
) -> tuple[float, list[RequestOutput] | None]:
55
- from vllm import LLM, SamplingParams
56
+ from vllm import LLM
57
58
llm = LLM.from_engine_args(engine_args)
59
+ all_requests = list(warmup_requests or []) + requests
60
assert all(
61
llm.llm_engine.model_config.max_model_len
62
>= (request.prompt_len + request.expected_output_len)
61
- for request in requests
63
+ for request in all_requests
64
), (
65
"Please ensure that max_model_len is greater than the sum of"
66
" prompt_len and expected_output_len for all requests."
67
)
66
- # Add the requests to the engine.
68
+
69
+ if warmup_requests:
70
+ print(f"Warming up with {len(warmup_requests)} requests...")
71
+ _run_vllm_requests(
72
+ llm,
73
+ warmup_requests,
74
+ n,
75
+ disable_detokenize,
76
+ do_profile=False,
77
+ prequeue_requests=prequeue_requests,
78
+ enable_lora=engine_args.enable_lora,
79
+ )
80
+
81
+ return _run_vllm_requests(
82
+ llm,
83
+ requests,
84
+ n,
85
+ disable_detokenize,
86
+ do_profile=do_profile,
87
+ prequeue_requests=prequeue_requests,
88
+ enable_lora=engine_args.enable_lora,
89
+ )
90
+
91
+
92
+def _run_vllm_requests(
93
+ llm: Any,
94
+ requests: list[SampleRequest],
95
+ n: int,
96
+ disable_detokenize: bool,
97
+ do_profile: bool,
98
+ prequeue_requests: bool,
99
+ enable_lora: bool,
100
+) -> tuple[float, list[RequestOutput] | None]:
101
+ from vllm import SamplingParams
102
+
103
prompts: list[TextPrompt | TokensPrompt] = []
104
sampling_params: list[SamplingParams] = []
105
+ lora_requests: list[LoRARequest] | None = [] if enable_lora else None
106
for request in requests:
107
prompt = (
108
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
@@ -87,9 +124,8 @@ def run_vllm(
124
detokenize=not disable_detokenize,
125
)
126
)
90
- lora_requests: list[LoRARequest] | None = None
91
- if engine_args.enable_lora:
92
- lora_requests = [request.lora_request for request in requests]
127
+ if lora_requests is not None:
128
+ lora_requests.append(request.lora_request)
129
130
use_beam_search = False
131
@@ -123,7 +159,7 @@ def run_vllm(
159
end = time.perf_counter()
160
else:
161
assert lora_requests is None, "BeamSearch API does not support LoRA"
126
- prompts = [request.prompt for request in requests]
162
+ beam_prompts = [request.prompt for request in requests]
163
# output_len should be the same for all requests.
164
output_len = requests[0].expected_output_len
165
for request in requests:
@@ -132,7 +168,7 @@ def run_vllm(
168
if do_profile:
169
llm.start_profile()
170
llm.beam_search(
135
- prompts,
171
+ beam_prompts,
172
BeamSearchParams(
173
beam_width=n,
174
max_tokens=output_len,
@@ -151,6 +187,7 @@ def run_vllm_chat(
187
engine_args: EngineArgs,
188
do_profile: bool,
189
disable_detokenize: bool = False,
190
+ warmup_requests: list[SampleRequest] | None = None,
191
prequeue_requests: bool = False,
192
) -> tuple[float, list[RequestOutput]]:
193
"""
@@ -158,23 +195,54 @@ def run_vllm_chat(
195
multimodal models as it properly handles multimodal inputs and chat
196
formatting. For non-multimodal models, use run_vllm() instead.
197
"""
161
- from vllm import LLM, SamplingParams
198
+ from vllm import LLM
199
200
llm = LLM.from_engine_args(engine_args)
201
202
+ all_requests = list(warmup_requests or []) + requests
203
assert all(
204
llm.llm_engine.model_config.max_model_len
205
>= (request.prompt_len + request.expected_output_len)
168
- for request in requests
206
+ for request in all_requests
207
), (
208
"Please ensure that max_model_len is greater than the sum of "
209
"prompt_len and expected_output_len for all requests."
210
)
211
174
- prompts = []
212
+ if warmup_requests:
213
+ print(f"Warming up with {len(warmup_requests)} requests...")
214
+ _run_vllm_chat_requests(
215
+ llm,
216
+ warmup_requests,
217
+ n,
218
+ disable_detokenize,
219
+ do_profile=False,
220
+ prequeue_requests=prequeue_requests,
221
+ )
222
+
223
+ return _run_vllm_chat_requests(
224
+ llm,
225
+ requests,
226
+ n,
227
+ disable_detokenize,
228
+ do_profile=do_profile,
229
+ prequeue_requests=prequeue_requests,
230
+ )
231
+
232
+
233
+def _run_vllm_chat_requests(
234
+ llm: Any,
235
+ requests: list[SampleRequest],
236
+ n: int,
237
+ disable_detokenize: bool,
238
+ do_profile: bool,
239
+ prequeue_requests: bool,
240
+) -> tuple[float, list[RequestOutput]]:
241
+ from vllm import SamplingParams
242
+
243
+ prompts = [request.prompt for request in requests]
244
sampling_params: list[SamplingParams] = []
245
for request in requests:
177
- prompts.append(request.prompt)
246
sampling_params.append(
247
SamplingParams(
248
n=n,
@@ -185,6 +253,7 @@ def run_vllm_chat(
253
detokenize=not disable_detokenize,
254
)
255
)
256
+
257
if prequeue_requests:
258
llm.sleep(level=0, mode="abort")
259
@@ -214,8 +283,8 @@ async def run_vllm_async(
283
engine_args: AsyncEngineArgs,
284
do_profile: bool,
285
disable_detokenize: bool = False,
286
+ warmup_requests: list[SampleRequest] | None = None,
287
) -> float:
218
- from vllm import SamplingParams
288
from vllm.entrypoints.openai.api_server import (
289
build_async_engine_client_from_engine_args,
290
)
@@ -224,59 +293,91 @@ async def run_vllm_async(
293
engine_args,
294
) as llm:
295
model_config = llm.model_config
296
+ all_requests = list(warmup_requests or []) + requests
297
assert all(
298
model_config.max_model_len
299
>= (request.prompt_len + request.expected_output_len)
230
- for request in requests
300
+ for request in all_requests
301
), (
302
"Please ensure that max_model_len is greater than the sum of"
303
" prompt_len and expected_output_len for all requests."
304
)
305
236
- # Add the requests to the engine.
237
- prompts: list[TextPrompt | TokensPrompt] = []
238
- sampling_params: list[SamplingParams] = []
239
- lora_requests: list[LoRARequest | None] = []
240
- for request in requests:
241
- prompt = (
242
- TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
243
- if "prompt_token_ids" in request.prompt
244
- else TextPrompt(prompt=request.prompt)
306
+ if warmup_requests:
307
+ print(f"Warming up with {len(warmup_requests)} requests...")
308
+ await _run_vllm_async_requests(
309
+ llm,
310
+ warmup_requests,
311
+ n,
312
+ disable_detokenize,
313
+ do_profile=False,
314
+ request_id_prefix="warmup",
315
)
316
247
- if request.multi_modal_data:
248
- assert isinstance(request.multi_modal_data, dict)
249
- prompt["multi_modal_data"] = request.multi_modal_data
250
-
251
- sampling_params.append(
252
- SamplingParams(
253
- n=n,
254
- temperature=1.0,
255
- top_p=1.0,
256
- ignore_eos=True,
257
- max_tokens=request.expected_output_len,
258
- detokenize=not disable_detokenize,
259
- )
317
+ elapsed_time, _ = await _run_vllm_async_requests(
318
+ llm,
319
+ requests,
320
+ n,
321
+ disable_detokenize,
322
+ do_profile=do_profile,
323
+ request_id_prefix="test",
324
+ )
325
+ return elapsed_time
326
+
327
+
328
+async def _run_vllm_async_requests(
329
+ llm: Any,
330
+ requests: list[SampleRequest],
331
+ n: int,
332
+ disable_detokenize: bool,
333
+ do_profile: bool,
334
+ request_id_prefix: str,
335
+) -> tuple[float, None]:
336
+ from vllm import SamplingParams
337
+
338
+ prompts: list[TextPrompt | TokensPrompt] = []
339
+ sampling_params: list[SamplingParams] = []
340
+ lora_requests: list[LoRARequest | None] = []
341
+ for request in requests:
342
+ prompt = (
343
+ TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
344
+ if "prompt_token_ids" in request.prompt
345
+ else TextPrompt(prompt=request.prompt)
346
+ )
347
+
348
+ if request.multi_modal_data:
349
+ assert isinstance(request.multi_modal_data, dict)
350
+ prompt["multi_modal_data"] = request.multi_modal_data
351
+
352
+ sampling_params.append(
353
+ SamplingParams(
354
+ n=n,
355
+ temperature=1.0,
356
+ top_p=1.0,
357
+ ignore_eos=True,
358
+ max_tokens=request.expected_output_len,
359
+ detokenize=not disable_detokenize,
360
)
261
- prompts.append(prompt)
262
- lora_requests.append(request.lora_request)
361
+ )
362
+ prompts.append(prompt)
363
+ lora_requests.append(request.lora_request)
364
264
- generators = []
265
- start = time.perf_counter()
266
- if do_profile:
267
- await llm.start_profile()
268
- for i, (prompt, sp, lr) in enumerate(
269
- zip(prompts, sampling_params, lora_requests)
270
- ):
271
- generator = llm.generate(prompt, sp, lora_request=lr, request_id=f"test{i}")
272
- generators.append(generator)
273
- all_gens = merge_async_iterators(*generators)
274
- async for i, res in all_gens:
275
- pass
276
- if do_profile:
277
- await llm.stop_profile()
278
- end = time.perf_counter()
279
- return end - start
365
+ generators = []
366
+ start = time.perf_counter()
367
+ if do_profile:
368
+ await llm.start_profile()
369
+ for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
370
+ generator = llm.generate(
371
+ prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
372
+ )
373
+ generators.append(generator)
374
+ all_gens = merge_async_iterators(*generators)
375
+ async for _i, _res in all_gens:
376
+ pass
377
+ if do_profile:
378
+ await llm.stop_profile()
379
+ end = time.perf_counter()
380
+ return end - start, None
381
382
383
def run_hf(
@@ -289,6 +390,7 @@ def run_hf(
390
disable_detokenize: bool = False,
391
dtype: torch.dtype | None = torch.float16,
392
enable_torch_compile: bool = False,
393
+ warmup_requests: list[SampleRequest] | None = None,
394
) -> float:
395
assert isinstance(tokenizer, PreTrainedTokenizerBase), (
396
"the hf backend only supports HF tokenizers"
@@ -303,6 +405,31 @@ def run_hf(
405
if enable_torch_compile:
406
llm = torch.compile(llm)
407
408
+ if warmup_requests:
409
+ print(f"Warming up with {len(warmup_requests)} requests...")
410
+ _run_hf_requests(
411
+ llm,
412
+ tokenizer,
413
+ warmup_requests,
414
+ n,
415
+ max_batch_size,
416
+ disable_detokenize,
417
+ )
418
+
419
+ elapsed_time, _ = _run_hf_requests(
420
+ llm, tokenizer, requests, n, max_batch_size, disable_detokenize
421
+ )
422
+ return elapsed_time
423
+
424
+
425
+def _run_hf_requests(
426
+ llm: Any,
427
+ tokenizer: PreTrainedTokenizerBase,
428
+ requests: list[SampleRequest],
429
+ n: int,
430
+ max_batch_size: int,
431
+ disable_detokenize: bool,
432
+) -> tuple[float, None]:
433
pbar = tqdm(total=len(requests))
434
start = time.perf_counter()
435
batch: list[str] = []
@@ -347,8 +474,9 @@ def run_hf(
474
batch = []
475
max_prompt_len = 0
476
max_output_len = 0
477
+ pbar.close()
478
end = time.perf_counter()
351
- return end - start
479
+ return end - start, None
480
481
482
def save_to_pytorch_benchmark_format(
@@ -801,6 +929,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
929
parser.add_argument(
930
"--num-prompts", type=int, default=1000, help="Number of prompts to process."
931
)
932
+ parser.add_argument(
933
+ "--num-warmups",
934
+ type=int,
935
+ default=0,
936
+ help="Number of warmup prompts to process before the timed benchmark.",
937
+ )
938
parser.add_argument(
939
"--hf-max-batch-size",
940
type=int,
@@ -971,6 +1105,14 @@ def main(args: argparse.Namespace):
1105
tokenizer_mode=args.tokenizer_mode,
1106
trust_remote_code=args.trust_remote_code,
1107
)
1108
+ num_warmups = args.num_warmups
1109
+ warmup_requests: list[SampleRequest] | None = None
1110
+ if num_warmups > 0:
1111
+ warmup_args = argparse.Namespace(**vars(args))
1112
+ warmup_args.num_prompts = num_warmups
1113
+ warmup_args.seed += 1
1114
+ warmup_requests = get_requests(warmup_args, tokenizer)
1115
+
1116
requests = get_requests(args, tokenizer)
1117
is_multi_modal = any(request.multi_modal_data is not None for request in requests)
1118
request_outputs: list[RequestOutput] | None = None
@@ -983,6 +1125,7 @@ def main(args: argparse.Namespace):
1125
AsyncEngineArgs.from_cli_args(args),
1126
disable_detokenize=args.disable_detokenize,
1127
do_profile=args.profile,
1128
+ warmup_requests=warmup_requests,
1129
)
1130
)
1131
else:
@@ -992,6 +1135,7 @@ def main(args: argparse.Namespace):
1135
EngineArgs.from_cli_args(args),
1136
disable_detokenize=args.disable_detokenize,
1137
do_profile=args.profile,
1138
+ warmup_requests=warmup_requests,
1139
prequeue_requests=args.prequeue_requests,
1140
)
1141
elif args.backend == "hf":
@@ -1008,6 +1152,7 @@ def main(args: argparse.Namespace):
1152
args.disable_detokenize,
1153
dtype=args.dtype,
1154
enable_torch_compile=args.hf_enable_torch_compile,
1155
+ warmup_requests=warmup_requests,
1156
)
1157
elif args.backend == "vllm-chat":
1158
elapsed_time, request_outputs = run_vllm_chat(
@@ -1016,6 +1161,7 @@ def main(args: argparse.Namespace):
1161
EngineArgs.from_cli_args(args),
1162
disable_detokenize=args.disable_detokenize,
1163
do_profile=args.profile,
1164
+ warmup_requests=warmup_requests,
1165
prequeue_requests=args.prequeue_requests,
1166
)
1167
else: