A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 06:45 UTC
add enqueue all option to throughput benchmark (#42975)
Signed-off-by: Philip Maybank <pmaybank@amd.com> Signed-off-by: pmaybank <113125070+pmaybank@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
pmaybank committed
May 19, 2026 at 23:16 UTC
fadf5d332c6e9bb6e552c1ca529511bce0f79802
3 files changed
+217
-7
tests/entrypoints/llm/test_mm_processor_kwargs.py
+45
@@ -76,6 +76,51 @@ def test_chat_forwards_mm_processor_kwargs() -> None:
76
)
77
78
79
+def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
80
+ llm = _make_mock_llm()
81
+ mm_processor_kwargs = {"do_pan_and_scan": True}
82
+ sampling_params = SamplingParams(max_tokens=1)
83
+ messages = [{"role": "user", "content": "hello"}]
84
+
85
+ llm._add_chat_requests = Mock(return_value=["req-0"])
86
+
87
+ request_ids = llm.enqueue_chat(
88
+ messages,
89
+ sampling_params=sampling_params,
90
+ use_tqdm=False,
91
+ mm_processor_kwargs=mm_processor_kwargs,
92
+ )
93
+
94
+ assert request_ids == ["req-0"]
95
+ assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
96
+ mm_processor_kwargs
97
+ )
98
+
99
+
100
+def test_run_chat_forwards_mm_processor_kwargs() -> None:
101
+ llm = _make_mock_llm()
102
+ mm_processor_kwargs = {"num_crops": 8}
103
+ sampling_params = SamplingParams(max_tokens=1)
104
+ messages = [{"role": "user", "content": "hello"}]
105
+ sentinel_output = ["done"]
106
+
107
+ llm._add_chat_requests = Mock()
108
+ llm._run_engine = Mock(return_value=sentinel_output)
109
+
110
+ outputs = llm._run_chat(
111
+ messages=messages,
112
+ params=sampling_params,
113
+ output_type=object,
114
+ use_tqdm=False,
115
+ mm_processor_kwargs=mm_processor_kwargs,
116
+ )
117
+
118
+ assert outputs == sentinel_output
119
+ assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
120
+ mm_processor_kwargs
121
+ )
122
+
123
+
124
def test_run_completion_forwards_mm_processor_kwargs() -> None:
125
llm = _make_mock_llm()
126
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
vllm/benchmarks/throughput.py
+56
-4
@@ -50,6 +50,7 @@ def run_vllm(
50
engine_args: EngineArgs,
51
do_profile: bool,
52
disable_detokenize: bool = False,
53
+ prequeue_requests: bool = False,
54
) -> tuple[float, list[RequestOutput] | None]:
55
from vllm import LLM, SamplingParams
56
@@ -94,12 +95,29 @@ def run_vllm(
95
96
outputs = None
97
if not use_beam_search:
98
+ if prequeue_requests:
99
+ llm.sleep(level=0, mode="abort")
100
+
101
start = time.perf_counter()
102
if do_profile:
103
llm.start_profile()
100
- outputs = llm.generate(
101
- prompts, sampling_params, lora_request=lora_requests, use_tqdm=True
102
- )
104
+
105
+ if prequeue_requests:
106
+ try:
107
+ llm.enqueue(
108
+ prompts,
109
+ sampling_params,
110
+ lora_request=lora_requests,
111
+ use_tqdm=True,
112
+ )
113
+ finally:
114
+ llm.wake_up(tags=["scheduling"])
115
+ outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
116
+ else:
117
+ outputs = llm.generate(
118
+ prompts, sampling_params, lora_request=lora_requests, use_tqdm=True
119
+ )
120
+
121
if do_profile:
122
llm.stop_profile()
123
end = time.perf_counter()
@@ -133,6 +151,7 @@ def run_vllm_chat(
151
engine_args: EngineArgs,
152
do_profile: bool,
153
disable_detokenize: bool = False,
154
+ prequeue_requests: bool = False,
155
) -> tuple[float, list[RequestOutput]]:
156
"""
157
Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
@@ -166,12 +185,25 @@ def run_vllm_chat(
185
detokenize=not disable_detokenize,
186
)
187
)
188
+ if prequeue_requests:
189
+ llm.sleep(level=0, mode="abort")
190
+
191
start = time.perf_counter()
192
if do_profile:
193
llm.start_profile()
172
- outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
194
+
195
+ if prequeue_requests:
196
+ try:
197
+ llm.enqueue_chat(prompts, sampling_params, use_tqdm=True)
198
+ finally:
199
+ llm.wake_up(tags=["scheduling"])
200
+ outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
201
+ else:
202
+ outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
203
+
204
if do_profile:
205
llm.stop_profile()
206
+
207
end = time.perf_counter()
208
return end - start, outputs
209
@@ -543,6 +575,10 @@ def validate_args(args):
575
valid_backends = {"vllm", "hf", "mii", "vllm-chat"}
576
if args.backend not in valid_backends:
577
raise ValueError(f"Unsupported backend: {args.backend}")
578
+ if args.prequeue_requests and args.backend not in {"vllm", "vllm-chat"}:
579
+ raise ValueError("--prequeue-requests requires --backend vllm or vllm-chat")
580
+ if args.prequeue_requests and args.async_engine:
581
+ raise ValueError("--prequeue-requests is not supported with --async-engine")
582
583
# === Dataset Configuration ===
584
if (
@@ -789,6 +825,20 @@ def add_cli_args(parser: argparse.ArgumentParser):
825
default=False,
826
help="Use vLLM async engine rather than LLM class.",
827
)
828
+ parser.add_argument(
829
+ "--prequeue-requests",
830
+ action="store_true",
831
+ default=False,
832
+ help=(
833
+ "For the vLLM backends, enqueue all requests before allowing the "
834
+ "scheduler to process them. This can improve benchmark "
835
+ "reproducibility by removing overlap between request rendering "
836
+ "and engine scheduling, but may reduce measured throughput. "
837
+ "Request rendering is typically fast relative to scheduling and "
838
+ "processing; the intended use case of this flag is multimodal "
839
+ "benchmarks with time-consuming image rendering."
840
+ ),
841
+ )
842
parser.add_argument(
843
"--disable-detokenize",
844
action="store_true",
@@ -942,6 +992,7 @@ def main(args: argparse.Namespace):
992
EngineArgs.from_cli_args(args),
993
disable_detokenize=args.disable_detokenize,
994
do_profile=args.profile,
995
+ prequeue_requests=args.prequeue_requests,
996
)
997
elif args.backend == "hf":
998
assert args.tensor_parallel_size == 1
@@ -965,6 +1016,7 @@ def main(args: argparse.Namespace):
1016
EngineArgs.from_cli_args(args),
1017
disable_detokenize=args.disable_detokenize,
1018
do_profile=args.profile,
1019
+ prequeue_requests=args.prequeue_requests,
1020
)
1021
else:
1022
raise ValueError(f"Unknown backend: {args.backend}")
vllm/entrypoints/llm.py
+116
-3
@@ -894,6 +894,83 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
894
mm_processor_kwargs=mm_processor_kwargs,
895
)
896
897
+ def enqueue_chat(
898
+ self,
899
+ messages: list[ChatCompletionMessageParam]
900
+ | Sequence[list[ChatCompletionMessageParam]],
901
+ sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
902
+ use_tqdm: bool | Callable[..., tqdm] = True,
903
+ lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
904
+ priority: list[int] | None = None,
905
+ chat_template: str | None = None,
906
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
907
+ add_generation_prompt: bool = True,
908
+ continue_final_message: bool = False,
909
+ tools: list[dict[str, Any]] | None = None,
910
+ chat_template_kwargs: dict[str, Any] | None = None,
911
+ tokenization_kwargs: dict[str, Any] | None = None,
912
+ mm_processor_kwargs: dict[str, Any] | None = None,
913
+ ) -> list[str]:
914
+ """Enqueue chat conversations for generation without waiting.
915
+
916
+ This method renders chat conversations and adds the resulting requests
917
+ to the engine queue. Use wait_for_completion() to get results. To
918
+ guarantee that all requests are queued before scheduling starts, pause
919
+ scheduling with sleep(level=0) before calling this method and resume it
920
+ with wake_up(tags=["scheduling"]) afterward.
921
+
922
+ Args:
923
+ messages: A sequence of conversations or a single conversation.
924
+ Each conversation is represented as a list of messages.
925
+ sampling_params: The sampling parameters for text generation.
926
+ If None, we use the default sampling parameters.
927
+ use_tqdm: If `True`, shows a tqdm progress bar while rendering
928
+ conversations.
929
+ lora_request: LoRA request to use for generation, if any.
930
+ priority: The priority of the requests, if any.
931
+ chat_template: The template to use for structuring the chat.
932
+ chat_template_content_format: The format to render message content.
933
+ add_generation_prompt: If True, adds a generation template
934
+ to each message.
935
+ continue_final_message: If True, continues the final message in
936
+ the conversation instead of starting a new one.
937
+ tools: Tools to make available to the model, if any.
938
+ chat_template_kwargs: Additional kwargs to pass to the chat
939
+ template.
940
+ tokenization_kwargs: Overrides for `tokenizer.encode`.
941
+ mm_processor_kwargs: Overrides for `processor.__call__`.
942
+
943
+ Returns:
944
+ A list of request IDs for the enqueued requests.
945
+ """
946
+ model_config = self.model_config
947
+ runner_type = model_config.runner_type
948
+ if runner_type != "generate":
949
+ raise ValueError(
950
+ "LLM.enqueue_chat() is only supported for generative models. "
951
+ "Try passing `--runner generate` to use the model as a "
952
+ "generative model."
953
+ )
954
+
955
+ if sampling_params is None:
956
+ sampling_params = self.get_default_sampling_params()
957
+
958
+ return self._add_chat_requests(
959
+ messages=messages,
960
+ params=sampling_params,
961
+ use_tqdm=use_tqdm,
962
+ lora_request=lora_request,
963
+ priority=priority,
964
+ chat_template=chat_template,
965
+ chat_template_content_format=chat_template_content_format,
966
+ chat_template_kwargs=chat_template_kwargs,
967
+ add_generation_prompt=add_generation_prompt,
968
+ continue_final_message=continue_final_message,
969
+ tools=tools,
970
+ tokenization_kwargs=tokenization_kwargs,
971
+ mm_processor_kwargs=mm_processor_kwargs,
972
+ )
973
+
974
def start_profile(self, profile_prefix: str | None = None) -> None:
975
"""Start profiling with optional custom trace prefix.
976
@@ -1095,9 +1172,46 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
1172
tokenization_kwargs: dict[str, Any] | None = None,
1173
mm_processor_kwargs: dict[str, Any] | None = None,
1174
):
1175
+ self._add_chat_requests(
1176
+ messages=messages,
1177
+ params=params,
1178
+ use_tqdm=use_tqdm,
1179
+ lora_request=lora_request,
1180
+ chat_template=chat_template,
1181
+ chat_template_content_format=chat_template_content_format,
1182
+ chat_template_kwargs=chat_template_kwargs,
1183
+ add_generation_prompt=add_generation_prompt,
1184
+ continue_final_message=continue_final_message,
1185
+ tools=tools,
1186
+ tokenization_kwargs=tokenization_kwargs,
1187
+ mm_processor_kwargs=mm_processor_kwargs,
1188
+ )
1189
+ return self._run_engine(output_type=output_type, use_tqdm=use_tqdm)
1190
+
1191
+ def _add_chat_requests(
1192
+ self,
1193
+ messages: list[ChatCompletionMessageParam]
1194
+ | Sequence[list[ChatCompletionMessageParam]],
1195
+ params: SamplingParams
1196
+ | PoolingParams
1197
+ | Sequence[SamplingParams | PoolingParams],
1198
+ *,
1199
+ use_tqdm: bool | Callable[..., tqdm] = True,
1200
+ lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1201
+ priority: list[int] | None = None,
1202
+ chat_template: str | None = None,
1203
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
1204
+ add_generation_prompt: bool = True,
1205
+ continue_final_message: bool = False,
1206
+ tools: list[dict[str, Any]] | None = None,
1207
+ chat_template_kwargs: dict[str, Any] | None = None,
1208
+ tokenization_kwargs: dict[str, Any] | None = None,
1209
+ mm_processor_kwargs: dict[str, Any] | None = None,
1210
+ ) -> list[str]:
1211
seq_convs = conversation_to_seq(messages)
1212
seq_params = self._params_to_seq(params, len(seq_convs))
1213
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
1214
+ seq_priority = self._priority_to_seq(priority, len(seq_convs))
1215
1216
# When thinking is enabled or tools are provided, and the model
1217
# uses special tokens for structured output (e.g. Gemma4's
@@ -1110,7 +1224,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
1224
if needs_parsing:
1225
self._adjust_params_for_parsing(seq_params)
1226
1113
- return self._render_and_run_requests(
1227
+ return self._render_and_add_requests(
1228
prompts=(
1229
self._preprocess_chat_one(
1230
conversation,
@@ -1130,9 +1244,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
1244
)
1245
),
1246
params=seq_params,
1133
- output_type=output_type,
1247
lora_requests=seq_lora_requests,
1135
- use_tqdm=use_tqdm,
1248
+ priorities=seq_priority,
1249
)
1250
1251
def _adjust_params_for_parsing(