| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for ``vllm bench throughput`` CLI. |
| 4 | |
| 5 | Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the |
| 6 | LoRA-assignment and MMVU cases below cover the pieces that are throughput-only |
| 7 | (serve has no analogue), keeping the two benchmarks aligned without duplicating |
| 8 | serve-side dataset coverage. |
| 9 | """ |
| 10 | |
| 11 | import subprocess |
| 12 | from argparse import Namespace |
| 13 | from types import SimpleNamespace |
| 14 | |
| 15 | import pytest |
| 16 | from transformers import AutoTokenizer, PreTrainedTokenizerBase |
| 17 | |
| 18 | from vllm.benchmarks.datasets import SampleRequest |
| 19 | from vllm.benchmarks.throughput import ( |
| 20 | _run_vllm_chat_requests, |
| 21 | _to_serve_args, |
| 22 | add_cli_args, |
| 23 | assign_loras, |
| 24 | get_requests, |
| 25 | ) |
| 26 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 27 | |
| 28 | MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct" |
| 29 | |
| 30 | |
| 31 | @pytest.mark.benchmark |
| 32 | def test_bench_throughput(): |
| 33 | command = [ |
| 34 | "vllm", |
| 35 | "bench", |
| 36 | "throughput", |
| 37 | "--model", |
| 38 | MODEL_NAME, |
| 39 | "--input-len", |
| 40 | "32", |
| 41 | "--output-len", |
| 42 | "1", |
| 43 | "--enforce-eager", |
| 44 | "--load-format", |
| 45 | "dummy", |
| 46 | ] |
| 47 | result = subprocess.run(command, capture_output=True, text=True) |
| 48 | print(result.stdout) |
| 49 | print(result.stderr) |
| 50 | |
| 51 | assert result.returncode == 0, f"Benchmark failed: {result.stderr}" |
| 52 | |
| 53 | |
| 54 | def test_bench_throughput_accepts_custom_audio_args(): |
| 55 | parser = FlexibleArgumentParser() |
| 56 | add_cli_args(parser) |
| 57 | |
| 58 | args = parser.parse_args( |
| 59 | [ |
| 60 | "--dataset-name", |
| 61 | "custom_audio", |
| 62 | "--dataset-path", |
| 63 | "audio.jsonl", |
| 64 | "--no-oversample", |
| 65 | "--custom-output-len", |
| 66 | "32", |
| 67 | "--enable-multimodal-chat", |
| 68 | ] |
| 69 | ) |
| 70 | |
| 71 | assert args.dataset_name == "custom_audio" |
| 72 | assert args.no_oversample |
| 73 | assert args.custom_output_len == 32 |
| 74 | assert args.enable_multimodal_chat |
| 75 | |
| 76 | |
| 77 | def test_vllm_chat_requests_include_multimodal_content(): |
| 78 | class FakeLLM: |
| 79 | def __init__(self): |
| 80 | self.prompts = None |
| 81 | |
| 82 | def chat(self, prompts, sampling_params, use_tqdm): |
| 83 | del sampling_params, use_tqdm |
| 84 | self.prompts = prompts |
| 85 | return [] |
| 86 | |
| 87 | llm = FakeLLM() |
| 88 | audio_content = { |
| 89 | "type": "input_audio", |
| 90 | "input_audio": {"data": "abc", "format": "wav"}, |
| 91 | } |
| 92 | request = SampleRequest( |
| 93 | prompt="Transcribe this audio.", |
| 94 | prompt_len=1, |
| 95 | expected_output_len=8, |
| 96 | multi_modal_data=audio_content, |
| 97 | ) |
| 98 | |
| 99 | _run_vllm_chat_requests( |
| 100 | llm, |
| 101 | [request], |
| 102 | n=1, |
| 103 | disable_detokenize=False, |
| 104 | do_profile=False, |
| 105 | prequeue_requests=False, |
| 106 | ) |
| 107 | |
| 108 | assert llm.prompts == [ |
| 109 | [ |
| 110 | { |
| 111 | "role": "user", |
| 112 | "content": [ |
| 113 | {"type": "text", "text": "Transcribe this audio."}, |
| 114 | audio_content, |
| 115 | ], |
| 116 | } |
| 117 | ] |
| 118 | ] |
| 119 | |
| 120 | |
| 121 | @pytest.fixture(scope="session") |
| 122 | def hf_tokenizer() -> PreTrainedTokenizerBase: |
| 123 | # Small, commonly available tokenizer. |
| 124 | return AutoTokenizer.from_pretrained("openai-community/gpt2") |
| 125 | |
| 126 | |
| 127 | # ----------------------------- |
| 128 | # LoRA assignment (throughput-only post-processing) |
| 129 | # ----------------------------- |
| 130 | |
| 131 | |
| 132 | def _sr(i: int) -> SampleRequest: |
| 133 | return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2) |
| 134 | |
| 135 | |
| 136 | def _lora_args( |
| 137 | *, |
| 138 | lora_path: str | None, |
| 139 | max_loras: int, |
| 140 | lora_assignment: str = "random", |
| 141 | ) -> SimpleNamespace: |
| 142 | return SimpleNamespace( |
| 143 | lora_path=lora_path, |
| 144 | max_loras=max_loras, |
| 145 | lora_assignment=lora_assignment, |
| 146 | ) |
| 147 | |
| 148 | |
| 149 | @pytest.mark.benchmark |
| 150 | def test_assign_loras_noop_without_lora_path() -> None: |
| 151 | """No lora_path -> requests returned unchanged, no LoRA attached.""" |
| 152 | reqs = [_sr(i) for i in range(5)] |
| 153 | out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4)) |
| 154 | assert out is reqs |
| 155 | assert all(r.lora_request is None for r in out) |
| 156 | |
| 157 | |
| 158 | @pytest.mark.benchmark |
| 159 | def test_assign_loras_round_robin() -> None: |
| 160 | """Round-robin assigns deterministic index % max_loras + 1 IDs.""" |
| 161 | reqs = [_sr(i) for i in range(6)] |
| 162 | out = assign_loras( |
| 163 | reqs, |
| 164 | _lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"), |
| 165 | ) |
| 166 | assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3] |
| 167 | |
| 168 | |
| 169 | @pytest.mark.benchmark |
| 170 | def test_assign_loras_random_in_range() -> None: |
| 171 | """Random assignment stays in [1, max_loras] and covers all IDs.""" |
| 172 | reqs = [_sr(i) for i in range(200)] |
| 173 | out = assign_loras( |
| 174 | reqs, |
| 175 | _lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"), |
| 176 | ) |
| 177 | ids = [r.lora_request.lora_int_id for r in out] |
| 178 | assert all(1 <= i <= 5 for i in ids) |
| 179 | assert len(set(ids)) == 5 |
| 180 | |
| 181 | |
| 182 | # ----------------------------- |
| 183 | # Issue #50838: MMVU via throughput |
| 184 | # ----------------------------- |
| 185 | |
| 186 | |
| 187 | @pytest.mark.benchmark |
| 188 | def test_get_requests_resolves_mmvu( |
| 189 | monkeypatch: pytest.MonkeyPatch, |
| 190 | hf_tokenizer: PreTrainedTokenizerBase, |
| 191 | ) -> None: |
| 192 | """Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput. |
| 193 | |
| 194 | Previously throughput's validate_args rejected MMVU ("is not supported by |
| 195 | hf dataset"). It now resolves through the shared get_samples. The HF |
| 196 | download is stubbed so the resolution + dispatch path is exercised without |
| 197 | network access. |
| 198 | """ |
| 199 | import vllm.benchmarks.datasets.datasets as dsmod |
| 200 | |
| 201 | class _StubMMVUDataset: |
| 202 | # MMVU is multimodal in content but, like the real class, does not set |
| 203 | # IS_MULTIMODAL -- so it is not subject to the backend gate. |
| 204 | IS_MULTIMODAL = False |
| 205 | SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]} |
| 206 | |
| 207 | def __init__(self, **kwargs): |
| 208 | self.__dict__.update(kwargs) |
| 209 | |
| 210 | def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002 |
| 211 | return [ |
| 212 | SampleRequest(prompt="q", prompt_len=1, expected_output_len=1) |
| 213 | for _ in range(num_requests) |
| 214 | ] |
| 215 | |
| 216 | monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset) |
| 217 | |
| 218 | parser = FlexibleArgumentParser() |
| 219 | add_cli_args(parser) |
| 220 | args = parser.parse_args( |
| 221 | [ |
| 222 | "--dataset-name", |
| 223 | "hf", |
| 224 | "--dataset-path", |
| 225 | "yale-nlp/MMVU", |
| 226 | "--backend", |
| 227 | "vllm-chat", |
| 228 | "--num-prompts", |
| 229 | "3", |
| 230 | ] |
| 231 | ) |
| 232 | requests = get_requests(args, hf_tokenizer) |
| 233 | assert len(requests) == 3 |
| 234 | assert all(isinstance(r, SampleRequest) for r in requests) |
| 235 | |
| 236 | |
| 237 | @pytest.mark.benchmark |
| 238 | def test_get_requests_allows_multimodal_on_plain_vllm_backend( |
| 239 | monkeypatch: pytest.MonkeyPatch, |
| 240 | hf_tokenizer: PreTrainedTokenizerBase, |
| 241 | ) -> None: |
| 242 | """Regression: --backend vllm must accept multimodal datasets such as ASR. |
| 243 | |
| 244 | _run_vllm_requests forwards multi_modal_data straight to LLM.generate, so |
| 245 | the vllm-chat-only gate in get_samples rejected a backend that always |
| 246 | worked. |
| 247 | """ |
| 248 | import vllm.benchmarks.datasets.datasets as dsmod |
| 249 | |
| 250 | class _StubASRDataset: |
| 251 | IS_MULTIMODAL = True |
| 252 | SUPPORTED_DATASET_PATHS = {"openslr/librispeech_asr"} |
| 253 | |
| 254 | def __init__(self, **kwargs): |
| 255 | self.__dict__.update(kwargs) |
| 256 | |
| 257 | def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002 |
| 258 | return [ |
| 259 | SampleRequest(prompt="q", prompt_len=1, expected_output_len=1) |
| 260 | for _ in range(num_requests) |
| 261 | ] |
| 262 | |
| 263 | monkeypatch.setattr(dsmod, "ASRDataset", _StubASRDataset) |
| 264 | |
| 265 | parser = FlexibleArgumentParser() |
| 266 | add_cli_args(parser) |
| 267 | args = parser.parse_args( |
| 268 | [ |
| 269 | "--dataset-name", |
| 270 | "hf", |
| 271 | "--dataset-path", |
| 272 | "openslr/librispeech_asr", |
| 273 | "--backend", |
| 274 | "vllm", |
| 275 | "--num-prompts", |
| 276 | "3", |
| 277 | ] |
| 278 | ) |
| 279 | requests = get_requests(args, hf_tokenizer) |
| 280 | assert len(requests) == 3 |
| 281 | assert all(isinstance(r, SampleRequest) for r in requests) |
| 282 | |
| 283 | |
| 284 | def test_to_serve_args_supports_mm_processor_flags() -> None: |
| 285 | """The adapter must serve callers without throughput's legacy flags. |
| 286 | |
| 287 | ``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len |
| 288 | nor --backend, and leaves --random-prefix-len at its default of 0. The |
| 289 | adapter must not read the legacy flags unconditionally, and must keep the |
| 290 | explicit 0 instead of coercing it to None (which later crashed |
| 291 | RandomMultiModalDataset.get_prefix on ``None <= 0``). |
| 292 | """ |
| 293 | args = Namespace( |
| 294 | random_input_len=32, |
| 295 | random_output_len=1, |
| 296 | random_prefix_len=0, |
| 297 | ) |
| 298 | serve_args = _to_serve_args(args) |
| 299 | |
| 300 | assert serve_args.random_input_len == 32 |
| 301 | assert serve_args.random_output_len == 1 |
| 302 | assert serve_args.random_prefix_len == 0 |
| 303 | assert serve_args.backend == "vllm-chat" |
| 304 | assert serve_args.enable_multimodal_chat |
| 305 | assert serve_args.no_oversample is False |
| 306 | |
| 307 | |
| 308 | def test_to_serve_args_keeps_legacy_flag_fallback() -> None: |
| 309 | """Callers that only set the legacy flags still get them mapped.""" |
| 310 | args = Namespace( |
| 311 | input_len=256, |
| 312 | output_len=8, |
| 313 | prefix_len=16, |
| 314 | backend="vllm", |
| 315 | ) |
| 316 | serve_args = _to_serve_args(args) |
| 317 | |
| 318 | assert serve_args.random_input_len == 256 |
| 319 | assert serve_args.random_output_len == 8 |
| 320 | assert serve_args.random_prefix_len == 16 |
| 321 | assert serve_args.backend == "vllm" |
| 322 | assert not serve_args.enable_multimodal_chat |
| 323 | |
| 324 | |
| 325 | @pytest.mark.benchmark |
| 326 | def test_get_requests_random_mm_with_mm_processor_args( |
| 327 | hf_tokenizer: PreTrainedTokenizerBase, |
| 328 | ) -> None: |
| 329 | """Regression: bench mm-processor's random-mm sampling must not crash. |
| 330 | |
| 331 | get_requests feeds mm-processor's parsed args (no legacy length/backend |
| 332 | flags) through the shared get_samples dispatch. With the default |
| 333 | --random-prefix-len 0 this previously crashed inside |
| 334 | RandomMultiModalDataset.get_prefix with a TypeError. |
| 335 | """ |
| 336 | from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args |
| 337 | |
| 338 | parser = FlexibleArgumentParser() |
| 339 | add_mm_cli_args(parser) |
| 340 | args = parser.parse_args( |
| 341 | [ |
| 342 | "--dataset-name", |
| 343 | "random-mm", |
| 344 | "--num-prompts", |
| 345 | "3", |
| 346 | "--random-input-len", |
| 347 | "32", |
| 348 | "--random-output-len", |
| 349 | "1", |
| 350 | "--random-mm-base-items-per-request", |
| 351 | "1", |
| 352 | "--random-mm-num-mm-items-range-ratio", |
| 353 | "0", |
| 354 | "--random-mm-limit-mm-per-prompt", |
| 355 | '{"image": 1}', |
| 356 | "--random-mm-bucket-config", |
| 357 | "{(224, 224, 1): 1.0}", |
| 358 | ] |
| 359 | ) |
| 360 | |
| 361 | requests = get_requests(args, hf_tokenizer) |
| 362 | assert len(requests) == 3 |
| 363 | assert all(isinstance(r, SampleRequest) for r in requests) |
| 364 | # mm-processor drives llm.chat, so prompts must be chat-formatted with |
| 365 | # multimodal content attached. |
| 366 | for req in requests: |
| 367 | assert isinstance(req.prompt, list) |
| 368 | assert req.prompt[0]["role"] == "user" |
| 369 | assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"]) |