[Bugfix][Bench] Fix bench mm-processor crash in shared request sampling (#56300)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Nicolò Lucchesi committed
Sep 11, 2026 at 10:53 UTC
9dcab802154d89d25de22332c458168673afab89
2 files changed
+110
-10
tests/benchmarks/test_throughput_cli.py
+89
@@ -17,6 +17,7 @@ from transformers import AutoTokenizer, PreTrainedTokenizerBase
17
from vllm.benchmarks.datasets import SampleRequest
18
from vllm.benchmarks.throughput import (
19
_run_vllm_chat_requests,
20
+ _to_serve_args,
21
add_cli_args,
22
assign_loras,
23
get_requests,
@@ -277,3 +278,91 @@ def test_get_requests_allows_multimodal_on_plain_vllm_backend(
278
requests = get_requests(args, hf_tokenizer)
279
assert len(requests) == 3
280
assert all(isinstance(r, SampleRequest) for r in requests)
281
+
282
+
283
+def test_to_serve_args_supports_mm_processor_flags() -> None:
284
+ """The adapter must serve callers without throughput's legacy flags.
285
+
286
+ ``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len
287
+ nor --backend, and leaves --random-prefix-len at its default of 0. The
288
+ adapter must not read the legacy flags unconditionally, and must keep the
289
+ explicit 0 instead of coercing it to None (which later crashed
290
+ RandomMultiModalDataset.get_prefix on ``None <= 0``).
291
+ """
292
+ args = SimpleNamespace(
293
+ random_input_len=32,
294
+ random_output_len=1,
295
+ random_prefix_len=0,
296
+ )
297
+ serve_args = _to_serve_args(args)
298
+
299
+ assert serve_args.random_input_len == 32
300
+ assert serve_args.random_output_len == 1
301
+ assert serve_args.random_prefix_len == 0
302
+ assert serve_args.backend == "vllm-chat"
303
+ assert serve_args.enable_multimodal_chat
304
+ assert serve_args.no_oversample is False
305
+
306
+
307
+def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
308
+ """Callers that only set the legacy flags still get them mapped."""
309
+ args = SimpleNamespace(
310
+ input_len=256,
311
+ output_len=8,
312
+ prefix_len=16,
313
+ backend="vllm",
314
+ )
315
+ serve_args = _to_serve_args(args)
316
+
317
+ assert serve_args.random_input_len == 256
318
+ assert serve_args.random_output_len == 8
319
+ assert serve_args.random_prefix_len == 16
320
+ assert serve_args.backend == "vllm"
321
+ assert not serve_args.enable_multimodal_chat
322
+
323
+
324
+@pytest.mark.benchmark
325
+def test_get_requests_random_mm_with_mm_processor_args(
326
+ hf_tokenizer: PreTrainedTokenizerBase,
327
+) -> None:
328
+ """Regression: bench mm-processor's random-mm sampling must not crash.
329
+
330
+ get_requests feeds mm-processor's parsed args (no legacy length/backend
331
+ flags) through the shared get_samples dispatch. With the default
332
+ --random-prefix-len 0 this previously crashed inside
333
+ RandomMultiModalDataset.get_prefix with a TypeError.
334
+ """
335
+ from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args
336
+
337
+ parser = FlexibleArgumentParser()
338
+ add_mm_cli_args(parser)
339
+ args = parser.parse_args(
340
+ [
341
+ "--dataset-name",
342
+ "random-mm",
343
+ "--num-prompts",
344
+ "3",
345
+ "--random-input-len",
346
+ "32",
347
+ "--random-output-len",
348
+ "1",
349
+ "--random-mm-base-items-per-request",
350
+ "1",
351
+ "--random-mm-num-mm-items-range-ratio",
352
+ "0",
353
+ "--random-mm-limit-mm-per-prompt",
354
+ '{"image": 1}',
355
+ "--random-mm-bucket-config",
356
+ "{(224, 224, 1): 1.0}",
357
+ ]
358
+ )
359
+
360
+ requests = get_requests(args, hf_tokenizer)
361
+ assert len(requests) == 3
362
+ assert all(isinstance(r, SampleRequest) for r in requests)
363
+ # mm-processor drives llm.chat, so prompts must be chat-formatted with
364
+ # multimodal content attached.
365
+ for req in requests:
366
+ assert isinstance(req.prompt, list)
367
+ assert req.prompt[0]["role"] == "user"
368
+ assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"])
vllm/benchmarks/throughput.py
+21
-10
@@ -539,27 +539,38 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
539
"""
540
d = vars(args).copy()
541
# random_*: prefer --random-* over legacy --input/output/prefix-len.
542
- d["random_input_len"] = getattr(args, "random_input_len", None) or args.input_len
543
- d["random_output_len"] = getattr(args, "random_output_len", None) or args.output_len
544
- d["random_prefix_len"] = getattr(args, "random_prefix_len", None) or args.prefix_len
542
+ # Callers without the legacy flags (e.g. bench mm-processor) omit them, so
543
+ # read them optionally. Identity checks, not truthiness: an explicit 0
544
+ # (the default of --random-prefix-len) must not fall through to them.
545
+ input_len = getattr(args, "input_len", None)
546
+ output_len = getattr(args, "output_len", None)
547
+ prefix_len = getattr(args, "prefix_len", None)
548
+ d["random_input_len"] = getattr(args, "random_input_len", input_len)
549
+ d["random_output_len"] = getattr(args, "random_output_len", output_len)
550
+ d["random_prefix_len"] = getattr(args, "random_prefix_len", prefix_len)
551
# --output-len maps to the per-dataset output-len entry points get_samples
552
# reads. None passes through (each dataset applies its own default),
553
# matching prior throughput behaviour of omitting output_len when unset.
548
- d["hf_output_len"] = args.output_len
549
- d["sharegpt_output_len"] = args.output_len
554
+ d["hf_output_len"] = output_len
555
+ d["sharegpt_output_len"] = output_len
556
# sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults.
551
- d["sonnet_input_len"] = args.input_len if args.input_len is not None else 550
552
- d["sonnet_output_len"] = args.output_len if args.output_len is not None else 150
553
- d["sonnet_prefix_len"] = args.prefix_len
557
+ d["sonnet_input_len"] = input_len if input_len is not None else 550
558
+ d["sonnet_output_len"] = output_len if output_len is not None else 150
559
+ d["sonnet_prefix_len"] = prefix_len
560
# Explicit --enable-multimodal-chat wins; otherwise auto-enable for the
555
- # multimodal chat backend (preserves today's vllm-chat handling).
561
+ # multimodal chat backend (preserves today's vllm-chat handling). Callers
562
+ # without a --backend flag (e.g. bench mm-processor) drive every request
563
+ # through llm.chat, so default to the vllm-chat behaviour.
564
+ backend = getattr(args, "backend", "vllm-chat")
565
+ d["backend"] = backend
566
d["enable_multimodal_chat"] = bool(
557
- getattr(args, "enable_multimodal_chat", False) or args.backend == "vllm-chat"
567
+ getattr(args, "enable_multimodal_chat", False) or backend == "vllm-chat"
568
)
569
# serve-only attrs throughput never exposed; keep serve's defaults.
570
d.setdefault("disable_shuffle", False)
571
d.setdefault("skip_chat_template", False)
572
d.setdefault("no_stream", False)
573
+ d.setdefault("no_oversample", False)
574
d.setdefault("request_id_prefix", "")
575
d.setdefault("chat_template_kwargs", None)
576
return argparse.Namespace(**d)