| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Benchmark offline inference throughput.""" |
| 4 | |
| 5 | import argparse |
| 6 | import json |
| 7 | import os |
| 8 | import random |
| 9 | import time |
| 10 | import warnings |
| 11 | from typing import Any |
| 12 | |
| 13 | import torch |
| 14 | import uvloop |
| 15 | from tqdm import tqdm |
| 16 | from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase |
| 17 | |
| 18 | from vllm.benchmarks.datasets import ( |
| 19 | BenchmarkDataset, |
| 20 | SampleRequest, |
| 21 | add_random_dataset_base_args, |
| 22 | add_random_multimodal_dataset_args, |
| 23 | get_samples, |
| 24 | ) |
| 25 | from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json |
| 26 | from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs |
| 27 | from vllm.inputs import TextPrompt, TokensPrompt |
| 28 | from vllm.lora.request import LoRARequest |
| 29 | from vllm.outputs import RequestOutput |
| 30 | from vllm.platforms import current_platform |
| 31 | from vllm.sampling_params import BeamSearchParams |
| 32 | from vllm.tokenizers import TokenizerLike, get_tokenizer |
| 33 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 34 | from vllm.utils.async_utils import merge_async_iterators |
| 35 | |
| 36 | |
| 37 | def run_vllm( |
| 38 | requests: list[SampleRequest], |
| 39 | n: int, |
| 40 | engine_args: EngineArgs, |
| 41 | do_profile: bool, |
| 42 | disable_detokenize: bool = False, |
| 43 | warmup_requests: list[SampleRequest] | None = None, |
| 44 | prequeue_requests: bool = False, |
| 45 | ) -> tuple[float, list[RequestOutput] | None]: |
| 46 | from vllm import LLM |
| 47 | |
| 48 | llm = LLM.from_engine_args(engine_args) |
| 49 | all_requests = list(warmup_requests or []) + requests |
| 50 | assert all( |
| 51 | llm.llm_engine.model_config.max_model_len |
| 52 | >= (request.prompt_len + request.expected_output_len) |
| 53 | for request in all_requests |
| 54 | ), ( |
| 55 | "Please ensure that max_model_len is greater than the sum of" |
| 56 | " prompt_len and expected_output_len for all requests." |
| 57 | ) |
| 58 | |
| 59 | if warmup_requests: |
| 60 | print(f"Warming up with {len(warmup_requests)} requests...") |
| 61 | _run_vllm_requests( |
| 62 | llm, |
| 63 | warmup_requests, |
| 64 | n, |
| 65 | disable_detokenize, |
| 66 | do_profile=False, |
| 67 | prequeue_requests=prequeue_requests, |
| 68 | enable_lora=engine_args.enable_lora, |
| 69 | ) |
| 70 | |
| 71 | return _run_vllm_requests( |
| 72 | llm, |
| 73 | requests, |
| 74 | n, |
| 75 | disable_detokenize, |
| 76 | do_profile=do_profile, |
| 77 | prequeue_requests=prequeue_requests, |
| 78 | enable_lora=engine_args.enable_lora, |
| 79 | ) |
| 80 | |
| 81 | |
| 82 | def _run_vllm_requests( |
| 83 | llm: Any, |
| 84 | requests: list[SampleRequest], |
| 85 | n: int, |
| 86 | disable_detokenize: bool, |
| 87 | do_profile: bool, |
| 88 | prequeue_requests: bool, |
| 89 | enable_lora: bool, |
| 90 | ) -> tuple[float, list[RequestOutput] | None]: |
| 91 | from vllm import SamplingParams |
| 92 | |
| 93 | prompts: list[TextPrompt | TokensPrompt] = [] |
| 94 | sampling_params: list[SamplingParams] = [] |
| 95 | lora_requests: list[LoRARequest | None] | None = [] if enable_lora else None |
| 96 | for request in requests: |
| 97 | if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt: |
| 98 | prompt_token_ids = request.prompt["prompt_token_ids"] |
| 99 | assert isinstance(prompt_token_ids, list) |
| 100 | prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) |
| 101 | else: |
| 102 | assert isinstance(request.prompt, str) |
| 103 | prompt = TextPrompt(prompt=request.prompt) |
| 104 | if request.multi_modal_data: |
| 105 | assert isinstance(request.multi_modal_data, dict) |
| 106 | prompt["multi_modal_data"] = request.multi_modal_data |
| 107 | prompts.append(prompt) |
| 108 | |
| 109 | sampling_params.append( |
| 110 | SamplingParams( |
| 111 | n=n, |
| 112 | temperature=1.0, |
| 113 | top_p=1.0, |
| 114 | ignore_eos=True, |
| 115 | max_tokens=request.expected_output_len, |
| 116 | detokenize=not disable_detokenize, |
| 117 | ) |
| 118 | ) |
| 119 | if lora_requests is not None: |
| 120 | lora_requests.append(request.lora_request) |
| 121 | |
| 122 | use_beam_search = False |
| 123 | |
| 124 | outputs = None |
| 125 | if not use_beam_search: |
| 126 | if prequeue_requests: |
| 127 | llm.sleep(level=0, mode="abort") |
| 128 | |
| 129 | start = time.perf_counter() |
| 130 | if do_profile: |
| 131 | llm.start_profile() |
| 132 | |
| 133 | if prequeue_requests: |
| 134 | try: |
| 135 | llm.enqueue( |
| 136 | prompts, |
| 137 | sampling_params, |
| 138 | lora_request=lora_requests, |
| 139 | use_tqdm=True, |
| 140 | ) |
| 141 | finally: |
| 142 | llm.wake_up(tags=["scheduling"]) |
| 143 | outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True) |
| 144 | else: |
| 145 | outputs = llm.generate( |
| 146 | prompts, sampling_params, lora_request=lora_requests, use_tqdm=True |
| 147 | ) |
| 148 | |
| 149 | if do_profile: |
| 150 | llm.stop_profile() |
| 151 | end = time.perf_counter() |
| 152 | else: |
| 153 | assert lora_requests is None, "BeamSearch API does not support LoRA" |
| 154 | beam_prompts: list[TextPrompt | TokensPrompt] = [] |
| 155 | for request in requests: |
| 156 | if isinstance(request.prompt, str): |
| 157 | beam_prompts.append(TextPrompt(prompt=request.prompt)) |
| 158 | elif ( |
| 159 | isinstance(request.prompt, dict) |
| 160 | and "prompt_token_ids" in request.prompt |
| 161 | ): |
| 162 | token_ids = request.prompt["prompt_token_ids"] |
| 163 | assert isinstance(token_ids, list) |
| 164 | beam_prompts.append(TokensPrompt(prompt_token_ids=token_ids)) |
| 165 | else: |
| 166 | # Fallback: convert to string |
| 167 | beam_prompts.append(TextPrompt(prompt=str(request.prompt))) |
| 168 | # output_len should be the same for all requests. |
| 169 | output_len = requests[0].expected_output_len |
| 170 | for request in requests: |
| 171 | assert request.expected_output_len == output_len |
| 172 | start = time.perf_counter() |
| 173 | if do_profile: |
| 174 | llm.start_profile() |
| 175 | llm.beam_search( |
| 176 | beam_prompts, |
| 177 | BeamSearchParams( |
| 178 | beam_width=n, |
| 179 | max_tokens=output_len, |
| 180 | ignore_eos=True, |
| 181 | ), |
| 182 | ) |
| 183 | if do_profile: |
| 184 | llm.stop_profile() |
| 185 | end = time.perf_counter() |
| 186 | return end - start, outputs |
| 187 | |
| 188 | |
| 189 | def run_vllm_chat( |
| 190 | requests: list[SampleRequest], |
| 191 | n: int, |
| 192 | engine_args: EngineArgs, |
| 193 | do_profile: bool, |
| 194 | disable_detokenize: bool = False, |
| 195 | warmup_requests: list[SampleRequest] | None = None, |
| 196 | prequeue_requests: bool = False, |
| 197 | ) -> tuple[float, list[RequestOutput]]: |
| 198 | """Run vLLM chat benchmark. This function is recommended ONLY for benchmarking |
| 199 | multimodal models as it properly handles multimodal inputs and chat |
| 200 | formatting. For non-multimodal models, use run_vllm() instead. |
| 201 | """ |
| 202 | from vllm import LLM |
| 203 | |
| 204 | llm = LLM.from_engine_args(engine_args) |
| 205 | |
| 206 | all_requests = list(warmup_requests or []) + requests |
| 207 | assert all( |
| 208 | llm.llm_engine.model_config.max_model_len |
| 209 | >= (request.prompt_len + request.expected_output_len) |
| 210 | for request in all_requests |
| 211 | ), ( |
| 212 | "Please ensure that max_model_len is greater than the sum of " |
| 213 | "prompt_len and expected_output_len for all requests." |
| 214 | ) |
| 215 | |
| 216 | if warmup_requests: |
| 217 | print(f"Warming up with {len(warmup_requests)} requests...") |
| 218 | _run_vllm_chat_requests( |
| 219 | llm, |
| 220 | warmup_requests, |
| 221 | n, |
| 222 | disable_detokenize, |
| 223 | do_profile=False, |
| 224 | prequeue_requests=prequeue_requests, |
| 225 | ) |
| 226 | |
| 227 | return _run_vllm_chat_requests( |
| 228 | llm, |
| 229 | requests, |
| 230 | n, |
| 231 | disable_detokenize, |
| 232 | do_profile=do_profile, |
| 233 | prequeue_requests=prequeue_requests, |
| 234 | ) |
| 235 | |
| 236 | |
| 237 | def _run_vllm_chat_requests( |
| 238 | llm: Any, |
| 239 | requests: list[SampleRequest], |
| 240 | n: int, |
| 241 | disable_detokenize: bool, |
| 242 | do_profile: bool, |
| 243 | prequeue_requests: bool, |
| 244 | ) -> tuple[float, list[RequestOutput]]: |
| 245 | from vllm import SamplingParams |
| 246 | |
| 247 | prompts = [] |
| 248 | sampling_params: list[SamplingParams] = [] |
| 249 | for request in requests: |
| 250 | if isinstance(request.prompt, list): |
| 251 | prompts.append(request.prompt) |
| 252 | else: |
| 253 | content: list[dict[str, Any]] = [{"type": "text", "text": request.prompt}] |
| 254 | if request.multi_modal_data is not None: |
| 255 | if isinstance(request.multi_modal_data, list): |
| 256 | content.extend(request.multi_modal_data) |
| 257 | elif isinstance(request.multi_modal_data, dict): |
| 258 | content.append(request.multi_modal_data) |
| 259 | else: |
| 260 | raise TypeError( |
| 261 | "Could not process multimodal content of type: " |
| 262 | f"{type(request.multi_modal_data)}" |
| 263 | ) |
| 264 | prompts.append([{"role": "user", "content": content}]) |
| 265 | sampling_params.append( |
| 266 | SamplingParams( |
| 267 | n=n, |
| 268 | temperature=1.0, |
| 269 | top_p=1.0, |
| 270 | ignore_eos=True, |
| 271 | max_tokens=request.expected_output_len, |
| 272 | detokenize=not disable_detokenize, |
| 273 | ) |
| 274 | ) |
| 275 | |
| 276 | if prequeue_requests: |
| 277 | llm.sleep(level=0, mode="abort") |
| 278 | |
| 279 | start = time.perf_counter() |
| 280 | if do_profile: |
| 281 | llm.start_profile() |
| 282 | |
| 283 | if prequeue_requests: |
| 284 | try: |
| 285 | llm.enqueue_chat(prompts, sampling_params, use_tqdm=True) |
| 286 | finally: |
| 287 | llm.wake_up(tags=["scheduling"]) |
| 288 | outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True) |
| 289 | else: |
| 290 | outputs = llm.chat(prompts, sampling_params, use_tqdm=True) # type: ignore[arg-type] |
| 291 | |
| 292 | if do_profile: |
| 293 | llm.stop_profile() |
| 294 | |
| 295 | end = time.perf_counter() |
| 296 | return end - start, outputs |
| 297 | |
| 298 | |
| 299 | async def run_vllm_async( |
| 300 | requests: list[SampleRequest], |
| 301 | n: int, |
| 302 | engine_args: AsyncEngineArgs, |
| 303 | do_profile: bool, |
| 304 | disable_detokenize: bool = False, |
| 305 | warmup_requests: list[SampleRequest] | None = None, |
| 306 | ) -> float: |
| 307 | from vllm.entrypoints.launchers.api_server.entry import ( |
| 308 | build_async_engine_client_from_engine_args, |
| 309 | ) |
| 310 | |
| 311 | async with build_async_engine_client_from_engine_args( |
| 312 | engine_args, |
| 313 | ) as llm: |
| 314 | model_config = llm.model_config |
| 315 | all_requests = list(warmup_requests or []) + requests |
| 316 | assert all( |
| 317 | model_config.max_model_len |
| 318 | >= (request.prompt_len + request.expected_output_len) |
| 319 | for request in all_requests |
| 320 | ), ( |
| 321 | "Please ensure that max_model_len is greater than the sum of" |
| 322 | " prompt_len and expected_output_len for all requests." |
| 323 | ) |
| 324 | |
| 325 | if warmup_requests: |
| 326 | print(f"Warming up with {len(warmup_requests)} requests...") |
| 327 | await _run_vllm_async_requests( |
| 328 | llm, |
| 329 | warmup_requests, |
| 330 | n, |
| 331 | disable_detokenize, |
| 332 | do_profile=False, |
| 333 | request_id_prefix="warmup", |
| 334 | ) |
| 335 | |
| 336 | elapsed_time, _ = await _run_vllm_async_requests( |
| 337 | llm, |
| 338 | requests, |
| 339 | n, |
| 340 | disable_detokenize, |
| 341 | do_profile=do_profile, |
| 342 | request_id_prefix="test", |
| 343 | ) |
| 344 | return elapsed_time |
| 345 | |
| 346 | |
| 347 | async def _run_vllm_async_requests( |
| 348 | llm: Any, |
| 349 | requests: list[SampleRequest], |
| 350 | n: int, |
| 351 | disable_detokenize: bool, |
| 352 | do_profile: bool, |
| 353 | request_id_prefix: str, |
| 354 | ) -> tuple[float, None]: |
| 355 | from vllm import SamplingParams |
| 356 | |
| 357 | prompts: list[TextPrompt | TokensPrompt] = [] |
| 358 | sampling_params: list[SamplingParams] = [] |
| 359 | lora_requests: list[LoRARequest | None] = [] |
| 360 | for request in requests: |
| 361 | if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt: |
| 362 | prompt_token_ids = request.prompt["prompt_token_ids"] |
| 363 | assert isinstance(prompt_token_ids, list) |
| 364 | prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) |
| 365 | else: |
| 366 | assert isinstance(request.prompt, str) |
| 367 | prompt = TextPrompt(prompt=request.prompt) |
| 368 | |
| 369 | if request.multi_modal_data: |
| 370 | assert isinstance(request.multi_modal_data, dict) |
| 371 | prompt["multi_modal_data"] = request.multi_modal_data |
| 372 | |
| 373 | sampling_params.append( |
| 374 | SamplingParams( |
| 375 | n=n, |
| 376 | temperature=1.0, |
| 377 | top_p=1.0, |
| 378 | ignore_eos=True, |
| 379 | max_tokens=request.expected_output_len, |
| 380 | detokenize=not disable_detokenize, |
| 381 | ) |
| 382 | ) |
| 383 | prompts.append(prompt) |
| 384 | lora_requests.append(request.lora_request) |
| 385 | |
| 386 | generators = [] |
| 387 | start = time.perf_counter() |
| 388 | if do_profile: |
| 389 | await llm.start_profile() |
| 390 | for i, (prompt_item, sp, lr) in enumerate( |
| 391 | zip(prompts, sampling_params, lora_requests) |
| 392 | ): |
| 393 | generator = llm.generate( |
| 394 | prompt_item, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}" |
| 395 | ) |
| 396 | generators.append(generator) |
| 397 | all_gens = merge_async_iterators(*generators) |
| 398 | async for _i, _res in all_gens: |
| 399 | pass |
| 400 | if do_profile: |
| 401 | await llm.stop_profile() |
| 402 | end = time.perf_counter() |
| 403 | return end - start, None |
| 404 | |
| 405 | |
| 406 | def run_hf( |
| 407 | requests: list[SampleRequest], |
| 408 | model: str, |
| 409 | tokenizer: TokenizerLike, |
| 410 | n: int, |
| 411 | max_batch_size: int, |
| 412 | trust_remote_code: bool, |
| 413 | disable_detokenize: bool = False, |
| 414 | dtype: torch.dtype | None = torch.float16, |
| 415 | enable_torch_compile: bool = False, |
| 416 | warmup_requests: list[SampleRequest] | None = None, |
| 417 | ) -> float: |
| 418 | assert isinstance(tokenizer, PreTrainedTokenizerBase), ( |
| 419 | "the hf backend only supports HF tokenizers" |
| 420 | ) |
| 421 | llm = AutoModelForCausalLM.from_pretrained( |
| 422 | model, dtype=dtype, trust_remote_code=trust_remote_code |
| 423 | ) |
| 424 | if llm.config.model_type == "llama": |
| 425 | # To enable padding in the HF backend. |
| 426 | tokenizer.pad_token = tokenizer.eos_token |
| 427 | llm = llm.to(current_platform.device_type) |
| 428 | if enable_torch_compile: |
| 429 | llm = torch.compile(llm) |
| 430 | |
| 431 | if warmup_requests: |
| 432 | print(f"Warming up with {len(warmup_requests)} requests...") |
| 433 | _run_hf_requests( |
| 434 | llm, |
| 435 | tokenizer, |
| 436 | warmup_requests, |
| 437 | n, |
| 438 | max_batch_size, |
| 439 | disable_detokenize, |
| 440 | ) |
| 441 | |
| 442 | elapsed_time, _ = _run_hf_requests( |
| 443 | llm, tokenizer, requests, n, max_batch_size, disable_detokenize |
| 444 | ) |
| 445 | return elapsed_time |
| 446 | |
| 447 | |
| 448 | def _run_hf_requests( |
| 449 | llm: Any, |
| 450 | tokenizer: PreTrainedTokenizerBase, |
| 451 | requests: list[SampleRequest], |
| 452 | n: int, |
| 453 | max_batch_size: int, |
| 454 | disable_detokenize: bool, |
| 455 | ) -> tuple[float, None]: |
| 456 | pbar = tqdm(total=len(requests)) |
| 457 | start = time.perf_counter() |
| 458 | batch: list[str] = [] |
| 459 | max_prompt_len = 0 |
| 460 | max_output_len = 0 |
| 461 | for i in range(len(requests)): |
| 462 | prompt = requests[i].prompt |
| 463 | prompt_len = requests[i].prompt_len |
| 464 | output_len = requests[i].expected_output_len |
| 465 | # Add the prompt to the batch. |
| 466 | assert isinstance(prompt, str), "Prompt must be a string for HF backend" |
| 467 | batch.append(prompt) |
| 468 | max_prompt_len = max(max_prompt_len, prompt_len) |
| 469 | max_output_len = max(max_output_len, output_len) |
| 470 | if len(batch) < max_batch_size and i != len(requests) - 1: |
| 471 | # Check if we can add more requests to the batch. |
| 472 | next_prompt_len = requests[i + 1].prompt_len |
| 473 | next_output_len = requests[i + 1].expected_output_len |
| 474 | if ( |
| 475 | max(max_prompt_len, next_prompt_len) |
| 476 | + max(max_output_len, next_output_len) |
| 477 | ) <= 2048: |
| 478 | # We can add more requests to the batch. |
| 479 | continue |
| 480 | |
| 481 | # Generate the sequences. |
| 482 | input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids |
| 483 | llm_outputs = llm.generate( |
| 484 | input_ids=input_ids.to(current_platform.device_type), |
| 485 | do_sample=True, |
| 486 | num_return_sequences=n, |
| 487 | temperature=1.0, |
| 488 | top_p=1.0, |
| 489 | use_cache=True, |
| 490 | max_new_tokens=max_output_len, |
| 491 | ) |
| 492 | if not disable_detokenize: |
| 493 | # Include the decoding time. |
| 494 | tokenizer.batch_decode(llm_outputs, skip_special_tokens=True) |
| 495 | pbar.update(len(batch)) |
| 496 | |
| 497 | # Clear the batch. |
| 498 | batch = [] |
| 499 | max_prompt_len = 0 |
| 500 | max_output_len = 0 |
| 501 | pbar.close() |
| 502 | end = time.perf_counter() |
| 503 | return end - start, None |
| 504 | |
| 505 | |
| 506 | def save_to_pytorch_benchmark_format( |
| 507 | args: argparse.Namespace, results: dict[str, Any] |
| 508 | ) -> None: |
| 509 | pt_records = convert_to_pytorch_benchmark_format( |
| 510 | args=args, |
| 511 | metrics={ |
| 512 | "requests_per_second": [results["requests_per_second"]], |
| 513 | "tokens_per_second": [results["tokens_per_second"]], |
| 514 | }, |
| 515 | extra_info={ |
| 516 | k: results[k] for k in ["elapsed_time", "num_requests", "total_num_tokens"] |
| 517 | }, |
| 518 | ) |
| 519 | if pt_records: |
| 520 | # Don't use json suffix here as we don't want CI to pick it up |
| 521 | pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json" |
| 522 | write_to_json(pt_file, pt_records) |
| 523 | |
| 524 | |
| 525 | def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace: |
| 526 | """Translate throughput args into a namespace the shared get_samples reads. |
| 527 | |
| 528 | ``get_samples`` (used by ``bench serve``) expects ~45 attributes; the |
| 529 | throughput CLI exposes most of them under the same names, and this adapter |
| 530 | fills the rest while preserving throughput's existing flag names so no |
| 531 | script breaks. |
| 532 | |
| 533 | Args: |
| 534 | args: parsed throughput CLI namespace. |
| 535 | |
| 536 | Returns: |
| 537 | A namespace satisfying get_samples's attribute reads. |
| 538 | |
| 539 | """ |
| 540 | d = vars(args).copy() |
| 541 | # random_*: prefer --random-* over legacy --input/output/prefix-len. |
| 542 | # Callers without the legacy flags (e.g. bench mm-processor) omit them, so |
| 543 | # read them optionally. Identity checks, not truthiness: an explicit 0 |
| 544 | # (the default of --random-prefix-len) must not fall through to them. |
| 545 | input_len = getattr(args, "input_len", None) |
| 546 | output_len = getattr(args, "output_len", None) |
| 547 | prefix_len = getattr(args, "prefix_len", None) |
| 548 | d["random_input_len"] = getattr(args, "random_input_len", input_len) |
| 549 | d["random_output_len"] = getattr(args, "random_output_len", output_len) |
| 550 | d["random_prefix_len"] = getattr(args, "random_prefix_len", prefix_len) |
| 551 | # --output-len maps to the per-dataset output-len entry points get_samples |
| 552 | # reads. None passes through (each dataset applies its own default), |
| 553 | # matching prior throughput behaviour of omitting output_len when unset. |
| 554 | d["hf_output_len"] = output_len |
| 555 | d["sharegpt_output_len"] = output_len |
| 556 | # sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults. |
| 557 | d["sonnet_input_len"] = input_len if input_len is not None else 550 |
| 558 | d["sonnet_output_len"] = output_len if output_len is not None else 150 |
| 559 | d["sonnet_prefix_len"] = prefix_len |
| 560 | # Explicit --enable-multimodal-chat wins; otherwise auto-enable for the |
| 561 | # multimodal chat backend (preserves today's vllm-chat handling). Callers |
| 562 | # without a --backend flag (e.g. bench mm-processor) drive every request |
| 563 | # through llm.chat, so default to the vllm-chat behaviour. |
| 564 | backend = getattr(args, "backend", "vllm-chat") |
| 565 | d["backend"] = backend |
| 566 | d["enable_multimodal_chat"] = bool( |
| 567 | getattr(args, "enable_multimodal_chat", False) or backend == "vllm-chat" |
| 568 | ) |
| 569 | # serve-only attrs throughput never exposed; keep serve's defaults. |
| 570 | d.setdefault("disable_shuffle", False) |
| 571 | d.setdefault("skip_chat_template", False) |
| 572 | d.setdefault("no_stream", False) |
| 573 | d.setdefault("no_oversample", False) |
| 574 | d.setdefault("request_id_prefix", "") |
| 575 | d.setdefault("chat_template_kwargs", None) |
| 576 | return argparse.Namespace(**d) |
| 577 | |
| 578 | |
| 579 | def get_requests(args, tokenizer): |
| 580 | serve_args = _to_serve_args(args) |
| 581 | # Both offline vLLM backends forward multi_modal_data to the engine; |
| 582 | # "hf" and "mii" are text-only and get rejected inside get_samples. |
| 583 | mm_backends = ("vllm", "vllm-chat") |
| 584 | requests = get_samples(serve_args, tokenizer, multimodal_backends=mm_backends) |
| 585 | requests = assign_loras(requests, args) |
| 586 | requests = filter_requests_for_dp(requests, args.data_parallel_size) |
| 587 | return requests |
| 588 | |
| 589 | |
| 590 | def filter_requests_for_dp(requests, data_parallel_size): |
| 591 | # Note(zhuohan): The way we get data_parallel_rank is hacky and only |
| 592 | # works for external launcher mode. Should be cleaned up and deprecated |
| 593 | # in the future with a better vLLM distributed process design. |
| 594 | if data_parallel_size == 1: |
| 595 | return requests |
| 596 | |
| 597 | global_rank = int(os.environ["RANK"]) |
| 598 | world_size = int(os.environ["WORLD_SIZE"]) |
| 599 | data_parallel_rank = global_rank // (world_size // data_parallel_size) |
| 600 | return [ |
| 601 | r |
| 602 | for i, r in enumerate(requests) |
| 603 | if i % data_parallel_size == data_parallel_rank |
| 604 | ] |
| 605 | |
| 606 | |
| 607 | def assign_loras(requests, args): |
| 608 | """Attach a LoRA request to each sample (throughput-only post-processing). |
| 609 | |
| 610 | The shared ``datasets.get_samples`` path carries no LoRA information, so |
| 611 | LoRA assignment is applied here, uniformly across every dataset type. No-op |
| 612 | when ``--lora-path`` is unset. |
| 613 | """ |
| 614 | lora_path = getattr(args, "lora_path", None) |
| 615 | if not lora_path: |
| 616 | return requests |
| 617 | max_loras = args.max_loras |
| 618 | lora_assignment = getattr(args, "lora_assignment", "random") |
| 619 | for i, req in enumerate(requests): |
| 620 | req.lora_request = BenchmarkDataset.get_lora_request( |
| 621 | index=i, |
| 622 | max_loras=max_loras, |
| 623 | lora_path=lora_path, |
| 624 | lora_assignment=lora_assignment, |
| 625 | ) |
| 626 | return requests |
| 627 | |
| 628 | |
| 629 | def validate_args(args): |
| 630 | """Validate command-line arguments.""" |
| 631 | # === Deprecation and Defaulting === |
| 632 | if args.dataset is not None: |
| 633 | warnings.warn( |
| 634 | "The '--dataset' argument will be deprecated in the next release. " |
| 635 | "Please use '--dataset-name' and '--dataset-path' instead.", |
| 636 | stacklevel=2, |
| 637 | ) |
| 638 | args.dataset_path = args.dataset |
| 639 | |
| 640 | if not getattr(args, "tokenizer", None): |
| 641 | args.tokenizer = args.model |
| 642 | |
| 643 | # === Backend Validation === |
| 644 | valid_backends = {"vllm", "hf", "mii", "vllm-chat"} |
| 645 | if args.backend not in valid_backends: |
| 646 | raise ValueError(f"Unsupported backend: {args.backend}") |
| 647 | if args.prequeue_requests and args.backend not in {"vllm", "vllm-chat"}: |
| 648 | raise ValueError("--prequeue-requests requires --backend vllm or vllm-chat") |
| 649 | if args.prequeue_requests and args.async_engine: |
| 650 | raise ValueError("--prequeue-requests is not supported with --async-engine") |
| 651 | |
| 652 | # === Dataset Configuration === |
| 653 | if ( |
| 654 | not args.dataset |
| 655 | and not args.dataset_path |
| 656 | and args.dataset_name not in {"prefix_repetition"} |
| 657 | ): |
| 658 | print("When dataset path is not set, it will default to random dataset") |
| 659 | args.dataset_name = "random" |
| 660 | random_input_len = getattr(args, "random_input_len", None) |
| 661 | if args.input_len is None and random_input_len is None: |
| 662 | raise ValueError( |
| 663 | "Either --input-len or --random-input-len must be provided " |
| 664 | "for a random dataset" |
| 665 | ) |
| 666 | |
| 667 | # === Dataset Name Specific Checks === |
| 668 | # --hf-subset and --hf-split: only used |
| 669 | # when dataset_name is 'hf' |
| 670 | if args.dataset_name != "hf" and ( |
| 671 | getattr(args, "hf_subset", None) is not None |
| 672 | or getattr(args, "hf_split", None) is not None |
| 673 | ): |
| 674 | warnings.warn( |
| 675 | "--hf-subset and --hf-split will be ignored \ |
| 676 | since --dataset-name is not 'hf'.", |
| 677 | stacklevel=2, |
| 678 | ) |
| 679 | |
| 680 | # --random-range-ratio: only used when dataset_name is 'random', |
| 681 | # 'random-mm', or 'random-rerank' |
| 682 | if ( |
| 683 | args.dataset_name not in {"random", "random-mm", "random-rerank"} |
| 684 | and args.random_range_ratio is not None |
| 685 | ): |
| 686 | warnings.warn( |
| 687 | "--random-range-ratio will be ignored since \ |
| 688 | --dataset-name is not 'random', 'random-mm', or 'random-rerank'.", |
| 689 | stacklevel=2, |
| 690 | ) |
| 691 | |
| 692 | # --random-batch-size: only used when dataset_name is 'random-rerank' |
| 693 | if ( |
| 694 | args.dataset_name != "random-rerank" |
| 695 | and getattr(args, "random_batch_size", None) is not None |
| 696 | ) and args.random_batch_size != 1: |
| 697 | warnings.warn( |
| 698 | "--random-batch-size will be ignored since \ |
| 699 | --dataset-name is not 'random-rerank'.", |
| 700 | stacklevel=2, |
| 701 | ) |
| 702 | |
| 703 | # --no-reranker: only used when dataset_name is 'random-rerank' |
| 704 | if args.dataset_name != "random-rerank" and getattr(args, "no_reranker", False): |
| 705 | warnings.warn( |
| 706 | "--no-reranker will be ignored since \ |
| 707 | --dataset-name is not 'random-rerank'.", |
| 708 | stacklevel=2, |
| 709 | ) |
| 710 | |
| 711 | # --prefix-len: only used when dataset_name is 'random', 'random-mm', |
| 712 | # 'sonnet', or not set. |
| 713 | if ( |
| 714 | args.dataset_name not in {"random", "random-mm", "sonnet", None} |
| 715 | and args.prefix_len is not None |
| 716 | ): |
| 717 | warnings.warn( |
| 718 | "--prefix-len will be ignored since --dataset-name\ |
| 719 | is not 'random', 'random-mm', 'sonnet', or not set.", |
| 720 | stacklevel=2, |
| 721 | ) |
| 722 | |
| 723 | # === Random Dataset Argument Conflict Detection === |
| 724 | # Check for conflicts between regular and random arguments when using |
| 725 | # random datasets |
| 726 | if args.dataset_name in {"random", "random-mm", "random-rerank"}: |
| 727 | random_input_len = getattr(args, "random_input_len", None) |
| 728 | random_output_len = getattr(args, "random_output_len", None) |
| 729 | random_prefix_len = getattr(args, "random_prefix_len", None) |
| 730 | |
| 731 | if args.input_len is not None and random_input_len is not None: |
| 732 | warnings.warn( |
| 733 | "Both --input-len and --random-input-len are specified. " |
| 734 | "The random version (--random-input-len) will be preferred " |
| 735 | "in this run.", |
| 736 | stacklevel=2, |
| 737 | ) |
| 738 | if args.output_len is not None and random_output_len is not None: |
| 739 | warnings.warn( |
| 740 | "Both --output-len and --random-output-len are specified. " |
| 741 | "The random version (--random-output-len) will be preferred " |
| 742 | "in this run.", |
| 743 | stacklevel=2, |
| 744 | ) |
| 745 | if args.prefix_len is not None and random_prefix_len is not None: |
| 746 | warnings.warn( |
| 747 | "Both --prefix-len and --random-prefix-len are specified. " |
| 748 | "The random version (--random-prefix-len) will be preferred " |
| 749 | "in this run.", |
| 750 | stacklevel=2, |
| 751 | ) |
| 752 | |
| 753 | # === LoRA Settings === |
| 754 | if getattr(args, "enable_lora", False) and args.backend != "vllm": |
| 755 | raise ValueError("LoRA benchmarking is only supported for vLLM backend") |
| 756 | if getattr(args, "enable_lora", False) and args.lora_path is None: |
| 757 | raise ValueError("LoRA path must be provided when enable_lora is True") |
| 758 | |
| 759 | # === Backend-specific Validations === |
| 760 | if args.backend == "hf" and args.hf_max_batch_size is None: |
| 761 | raise ValueError("HF max batch size is required for HF backend") |
| 762 | if args.backend != "hf" and args.hf_max_batch_size is not None: |
| 763 | raise ValueError("HF max batch size is only for HF backend.") |
| 764 | |
| 765 | if ( |
| 766 | args.backend in {"hf", "mii"} |
| 767 | and getattr(args, "quantization", None) is not None |
| 768 | ): |
| 769 | raise ValueError("Quantization is only for vLLM backend.") |
| 770 | |
| 771 | if args.backend == "mii" and args.dtype != "auto": |
| 772 | raise ValueError("dtype must be auto for MII backend.") |
| 773 | if args.backend == "mii" and args.n != 1: |
| 774 | raise ValueError("n must be 1 for MII backend.") |
| 775 | if args.backend == "mii" and args.tokenizer != args.model: |
| 776 | raise ValueError("Tokenizer must be the same as the model for MII backend.") |
| 777 | |
| 778 | if args.data_parallel_size > 1 and ( |
| 779 | args.distributed_executor_backend != "external_launcher" or args.async_engine |
| 780 | ): |
| 781 | # --data-parallel is not supported fully. |
| 782 | # Old issue: https://github.com/vllm-project/vllm/issues/16222 |
| 783 | # Currently we only support data parallel with external launcher |
| 784 | # mode (i.e., launch with toruchrun). |
| 785 | raise ValueError( |
| 786 | "Data parallel is only supported with external launcher mode " |
| 787 | "with synchronous engine in offline benchmark, " |
| 788 | "please use benchmark serving instead" |
| 789 | ) |
| 790 | |
| 791 | |
| 792 | def add_cli_args(parser: FlexibleArgumentParser): |
| 793 | parser.add_argument( |
| 794 | "--backend", |
| 795 | type=str, |
| 796 | choices=["vllm", "hf", "mii", "vllm-chat"], |
| 797 | default="vllm", |
| 798 | ) |
| 799 | parser.add_argument( |
| 800 | "--dataset-name", |
| 801 | type=str, |
| 802 | choices=[ |
| 803 | "sharegpt", |
| 804 | "random", |
| 805 | "sonnet", |
| 806 | "burstgpt", |
| 807 | "hf", |
| 808 | "prefix_repetition", |
| 809 | "random-mm", |
| 810 | "random-rerank", |
| 811 | "custom_audio", |
| 812 | ], |
| 813 | help="Name of the dataset to benchmark on.", |
| 814 | default="sharegpt", |
| 815 | ) |
| 816 | parser.add_argument( |
| 817 | "--dataset", |
| 818 | type=str, |
| 819 | default=None, |
| 820 | help="Path to the ShareGPT dataset, will be deprecated in\ |
| 821 | the next release. The dataset is expected to " |
| 822 | "be a json in form of list[dict[..., conversations: " |
| 823 | "list[dict[..., value: <prompt_or_response>]]]]", |
| 824 | ) |
| 825 | parser.add_argument( |
| 826 | "--dataset-path", type=str, default=None, help="Path to the dataset" |
| 827 | ) |
| 828 | parser.add_argument( |
| 829 | "--no-oversample", |
| 830 | action="store_true", |
| 831 | help="Do not oversample if the dataset has fewer samples than num-prompts.", |
| 832 | ) |
| 833 | parser.add_argument( |
| 834 | "--enable-multimodal-chat", |
| 835 | action="store_true", |
| 836 | help="Enable multimodal chat transformation for datasets that support it.", |
| 837 | ) |
| 838 | parser.add_argument( |
| 839 | "--custom-output-len", |
| 840 | type=int, |
| 841 | default=None, |
| 842 | help="Number of output tokens per request for custom datasets.", |
| 843 | ) |
| 844 | parser.add_argument( |
| 845 | "--input-len", |
| 846 | type=int, |
| 847 | default=None, |
| 848 | help="Input prompt length for each request", |
| 849 | ) |
| 850 | parser.add_argument( |
| 851 | "--output-len", |
| 852 | type=int, |
| 853 | default=None, |
| 854 | help="Output length for each request. Overrides the " |
| 855 | "output length from the dataset.", |
| 856 | ) |
| 857 | parser.add_argument( |
| 858 | "--n", type=int, default=1, help="Number of generated sequences per prompt." |
| 859 | ) |
| 860 | parser.add_argument( |
| 861 | "--num-prompts", type=int, default=1000, help="Number of prompts to process." |
| 862 | ) |
| 863 | parser.add_argument( |
| 864 | "--num-warmups", |
| 865 | type=int, |
| 866 | default=0, |
| 867 | help="Number of warmup prompts to process before the timed benchmark.", |
| 868 | ) |
| 869 | parser.add_argument( |
| 870 | "--hf-max-batch-size", |
| 871 | type=int, |
| 872 | default=None, |
| 873 | help="Maximum batch size for HF backend.", |
| 874 | ) |
| 875 | parser.add_argument( |
| 876 | "--hf-enable-torch-compile", |
| 877 | action="store_true", |
| 878 | default=False, |
| 879 | help="Enable Torch compile for HF backend.", |
| 880 | ) |
| 881 | parser.add_argument( |
| 882 | "--output-json", |
| 883 | type=str, |
| 884 | default=None, |
| 885 | help="Path to save the throughput results in JSON format.", |
| 886 | ) |
| 887 | parser.add_argument( |
| 888 | "--async-engine", |
| 889 | action="store_true", |
| 890 | default=False, |
| 891 | help="Use vLLM async engine rather than LLM class.", |
| 892 | ) |
| 893 | parser.add_argument( |
| 894 | "--prequeue-requests", |
| 895 | action="store_true", |
| 896 | default=False, |
| 897 | help=( |
| 898 | "For the vLLM backends, enqueue all requests before allowing the " |
| 899 | "scheduler to process them. This can improve benchmark " |
| 900 | "reproducibility by removing overlap between request rendering " |
| 901 | "and engine scheduling, but may reduce measured throughput. " |
| 902 | "Request rendering is typically fast relative to scheduling and " |
| 903 | "processing; the intended use case of this flag is multimodal " |
| 904 | "benchmarks with time-consuming image rendering." |
| 905 | ), |
| 906 | ) |
| 907 | parser.add_argument( |
| 908 | "--disable-detokenize", |
| 909 | action="store_true", |
| 910 | help=( |
| 911 | "Do not detokenize the response (i.e. do not include " |
| 912 | "detokenization time in the measurement)" |
| 913 | ), |
| 914 | ) |
| 915 | # LoRA |
| 916 | parser.add_argument( |
| 917 | "--lora-path", |
| 918 | type=str, |
| 919 | default=None, |
| 920 | help="Path to the lora adapters to use. This can be an absolute path, " |
| 921 | "a relative path, or a Hugging Face model identifier.", |
| 922 | ) |
| 923 | parser.add_argument( |
| 924 | "--lora-assignment", |
| 925 | type=str, |
| 926 | default="random", |
| 927 | choices=["random", "round-robin"], |
| 928 | help="Strategy for assigning LoRA adapters to requests. " |
| 929 | "'random' (default) selects a LoRA at random for each request. " |
| 930 | "'round-robin' cycles through LoRAs deterministically.", |
| 931 | ) |
| 932 | parser.add_argument( |
| 933 | "--prefix-len", |
| 934 | type=int, |
| 935 | default=0, |
| 936 | help="Number of fixed prefix tokens before the random " |
| 937 | "context in a request (default: 0).", |
| 938 | ) |
| 939 | |
| 940 | # hf dataset |
| 941 | parser.add_argument( |
| 942 | "--hf-subset", |
| 943 | type=str, |
| 944 | default=None, |
| 945 | help="Subset of the HF dataset.", |
| 946 | ) |
| 947 | parser.add_argument( |
| 948 | "--hf-split", |
| 949 | type=str, |
| 950 | default=None, |
| 951 | help="Split of the HF dataset.", |
| 952 | ) |
| 953 | parser.add_argument( |
| 954 | "--hf-name", |
| 955 | type=str, |
| 956 | default=None, |
| 957 | help=( |
| 958 | "Name of the dataset on HuggingFace " |
| 959 | "(e.g., 'lmms-lab/LLaVA-OneVision-Data'). " |
| 960 | "Specify this when --dataset-path is a local filesystem path " |
| 961 | "so the benchmark can identify the correct dataset class." |
| 962 | ), |
| 963 | ) |
| 964 | parser.add_argument( |
| 965 | "--profile", |
| 966 | action="store_true", |
| 967 | default=False, |
| 968 | help="Use vLLM Profiling. --profiler-config must be provided on the server.", |
| 969 | ) |
| 970 | |
| 971 | # prefix repetition dataset |
| 972 | parser.add_argument( |
| 973 | "--prefix-repetition-prefix-len", |
| 974 | type=int, |
| 975 | default=None, |
| 976 | help="Number of prefix tokens per request, used only for prefix " |
| 977 | "repetition dataset.", |
| 978 | ) |
| 979 | parser.add_argument( |
| 980 | "--prefix-repetition-suffix-len", |
| 981 | type=int, |
| 982 | default=None, |
| 983 | help="Number of suffix tokens per request, used only for prefix " |
| 984 | "repetition dataset. Total input length is prefix_len + suffix_len.", |
| 985 | ) |
| 986 | parser.add_argument( |
| 987 | "--prefix-repetition-num-prefixes", |
| 988 | type=int, |
| 989 | default=None, |
| 990 | help="Number of prefixes to generate, used only for prefix repetition " |
| 991 | "dataset. Prompts per prefix is num_requests // num_prefixes.", |
| 992 | ) |
| 993 | parser.add_argument( |
| 994 | "--prefix-repetition-output-len", |
| 995 | type=int, |
| 996 | default=None, |
| 997 | help="Number of output tokens per request, used only for prefix " |
| 998 | "repetition dataset.", |
| 999 | ) |
| 1000 | |
| 1001 | # (random, random-mm, random-rerank) |
| 1002 | add_random_dataset_base_args(parser) |
| 1003 | add_random_multimodal_dataset_args(parser) |
| 1004 | |
| 1005 | # ASR dataset |
| 1006 | parser.add_argument( |
| 1007 | "--asr-min-audio-len-sec", |
| 1008 | type=float, |
| 1009 | default=0.0, |
| 1010 | help="Minimum audio duration in seconds for ASR dataset filtering.", |
| 1011 | ) |
| 1012 | parser.add_argument( |
| 1013 | "--asr-max-audio-len-sec", |
| 1014 | type=float, |
| 1015 | default=float("inf"), |
| 1016 | help="Maximum audio duration in seconds for ASR dataset filtering.", |
| 1017 | ) |
| 1018 | |
| 1019 | parser = AsyncEngineArgs.add_cli_args(parser) |
| 1020 | |
| 1021 | |
| 1022 | def main(args: argparse.Namespace): |
| 1023 | validate_args(args) |
| 1024 | if args.seed is None: |
| 1025 | args.seed = 0 |
| 1026 | random.seed(args.seed) |
| 1027 | # Sample the requests. |
| 1028 | if ( |
| 1029 | args.backend == "hf" or args.backend == "mii" |
| 1030 | ) and args.tokenizer_mode == "auto": |
| 1031 | # mistral_common tokenizer is only supported on vllm and vllm-chat backends; |
| 1032 | # for hf and mii backends, we use hf tokenizer |
| 1033 | args.tokenizer_mode = "hf" |
| 1034 | tokenizer = get_tokenizer( |
| 1035 | args.tokenizer, |
| 1036 | tokenizer_mode=args.tokenizer_mode, |
| 1037 | trust_remote_code=args.trust_remote_code, |
| 1038 | ) |
| 1039 | num_warmups = args.num_warmups |
| 1040 | warmup_requests: list[SampleRequest] | None = None |
| 1041 | if num_warmups > 0: |
| 1042 | warmup_args = argparse.Namespace(**vars(args)) |
| 1043 | warmup_args.num_prompts = num_warmups |
| 1044 | warmup_args.seed += 1 |
| 1045 | warmup_requests = get_requests(warmup_args, tokenizer) |
| 1046 | |
| 1047 | requests = get_requests(args, tokenizer) |
| 1048 | is_multi_modal = any(request.multi_modal_data is not None for request in requests) |
| 1049 | request_outputs: list[RequestOutput] | None = None |
| 1050 | if args.backend == "vllm": |
| 1051 | if args.async_engine: |
| 1052 | elapsed_time = uvloop.run( |
| 1053 | run_vllm_async( |
| 1054 | requests, |
| 1055 | args.n, |
| 1056 | AsyncEngineArgs.from_cli_args(args), |
| 1057 | disable_detokenize=args.disable_detokenize, |
| 1058 | do_profile=args.profile, |
| 1059 | warmup_requests=warmup_requests, |
| 1060 | ) |
| 1061 | ) |
| 1062 | else: |
| 1063 | elapsed_time, request_outputs = run_vllm( |
| 1064 | requests, |
| 1065 | args.n, |
| 1066 | EngineArgs.from_cli_args(args), |
| 1067 | disable_detokenize=args.disable_detokenize, |
| 1068 | do_profile=args.profile, |
| 1069 | warmup_requests=warmup_requests, |
| 1070 | prequeue_requests=args.prequeue_requests, |
| 1071 | ) |
| 1072 | elif args.backend == "hf": |
| 1073 | assert args.tensor_parallel_size == 1 |
| 1074 | if args.profile: |
| 1075 | raise NotImplementedError("Profiling not implemented yet for backend='hf'.") |
| 1076 | elapsed_time = run_hf( |
| 1077 | requests, |
| 1078 | args.model, |
| 1079 | tokenizer, |
| 1080 | args.n, |
| 1081 | args.hf_max_batch_size, |
| 1082 | args.trust_remote_code, |
| 1083 | args.disable_detokenize, |
| 1084 | dtype=args.dtype, |
| 1085 | enable_torch_compile=args.hf_enable_torch_compile, |
| 1086 | warmup_requests=warmup_requests, |
| 1087 | ) |
| 1088 | elif args.backend == "vllm-chat": |
| 1089 | elapsed_time, request_outputs = run_vllm_chat( |
| 1090 | requests, |
| 1091 | args.n, |
| 1092 | EngineArgs.from_cli_args(args), |
| 1093 | disable_detokenize=args.disable_detokenize, |
| 1094 | do_profile=args.profile, |
| 1095 | warmup_requests=warmup_requests, |
| 1096 | prequeue_requests=args.prequeue_requests, |
| 1097 | ) |
| 1098 | else: |
| 1099 | raise ValueError(f"Unknown backend: {args.backend}") |
| 1100 | |
| 1101 | if request_outputs: |
| 1102 | # Note: with the vllm and vllm-chat backends, |
| 1103 | # we have request_outputs, which we use to count tokens. |
| 1104 | total_prompt_tokens = 0 |
| 1105 | total_output_tokens = 0 |
| 1106 | for ro in request_outputs: |
| 1107 | if not isinstance(ro, RequestOutput): |
| 1108 | continue |
| 1109 | total_prompt_tokens += ( |
| 1110 | len(ro.prompt_token_ids) if ro.prompt_token_ids else 0 |
| 1111 | ) |
| 1112 | total_output_tokens += sum( |
| 1113 | len(o.token_ids) |
| 1114 | for o in ro.outputs |
| 1115 | if o is not None and o.token_ids is not None |
| 1116 | ) |
| 1117 | total_num_tokens = total_prompt_tokens + total_output_tokens |
| 1118 | else: |
| 1119 | total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests) |
| 1120 | total_output_tokens = sum(r.expected_output_len for r in requests) |
| 1121 | total_prompt_tokens = total_num_tokens - total_output_tokens |
| 1122 | |
| 1123 | if is_multi_modal and args.backend != "vllm-chat": |
| 1124 | print( |
| 1125 | "\033[91mWARNING\033[0m: Multi-modal request with " |
| 1126 | f"{args.backend} backend detected. The " |
| 1127 | "following metrics are not accurate because image tokens are not" |
| 1128 | " counted. See vllm-project/vllm/issues/9778 for details." |
| 1129 | ) |
| 1130 | # TODO(vllm-project/vllm/issues/9778): Count multi-modal token length. |
| 1131 | # vllm-chat backend counts the image tokens now |
| 1132 | |
| 1133 | print( |
| 1134 | f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, " |
| 1135 | f"{total_num_tokens / elapsed_time:.2f} total tokens/s, " |
| 1136 | f"{total_output_tokens / elapsed_time:.2f} output tokens/s" |
| 1137 | ) |
| 1138 | print(f"Total num prompt tokens: {total_prompt_tokens}") |
| 1139 | print(f"Total num output tokens: {total_output_tokens}") |
| 1140 | |
| 1141 | # Output JSON results if specified |
| 1142 | if args.output_json: |
| 1143 | results = { |
| 1144 | "model_id": args.model, |
| 1145 | "elapsed_time": elapsed_time, |
| 1146 | "num_requests": len(requests), |
| 1147 | "total_num_tokens": total_num_tokens, |
| 1148 | "requests_per_second": len(requests) / elapsed_time, |
| 1149 | "tokens_per_second": total_num_tokens / elapsed_time, |
| 1150 | } |
| 1151 | with open(args.output_json, "w") as f: |
| 1152 | json.dump(results, f, indent=4) |
| 1153 | save_to_pytorch_benchmark_format(args, results) |