@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b4fe16c75b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

Add `vllm bench [latency, throughput]` CLI commands (#16508)

Signed-off-by: mgoin <mgoin64@gmail.com>

Michael Goin committed Apr 15, 2025 at 00:10 UTC b4fe16c75b437794900afcc3e1aa53df34e5ea38
11 files changed +1771 -2
.buildkite/test-pipeline.yaml
+7
@@ -341,6 +341,13 @@ steps:
341 commands:
342 - bash scripts/run-benchmarks.sh
343
344 +- label: Benchmarks CLI Test # 10min
345 + source_file_dependencies:
346 + - vllm/
347 + - tests/benchmarks/
348 + commands:
349 + - pytest -v -s benchmarks/
350 +
351 - label: Quantization Test # 33min
352 source_file_dependencies:
353 - csrc/
tests/benchmarks/__init__.py
tests/benchmarks/test_latency_cli.py new
+19
@@ -0,0 +1,19 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import subprocess
3 +
4 +import pytest
5 +
6 +MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
7 +
8 +
9 +@pytest.mark.benchmark
10 +def test_bench_latency():
11 + command = [
12 + "vllm", "bench", "latency", "--model", MODEL_NAME, "--input-len", "32",
13 + "--output-len", "1", "--enforce-eager", "--load-format", "dummy"
14 + ]
15 + result = subprocess.run(command, capture_output=True, text=True)
16 + print(result.stdout)
17 + print(result.stderr)
18 +
19 + assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
tests/benchmarks/test_serve_cli.py new
+44
@@ -0,0 +1,44 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import subprocess
3 +
4 +import pytest
5 +
6 +from ..utils import RemoteOpenAIServer
7 +
8 +MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
9 +
10 +
11 +@pytest.fixture(scope="module")
12 +def server():
13 + args = [
14 + "--max-model-len", "1024", "--enforce-eager", "--load-format", "dummy"
15 + ]
16 +
17 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
18 + yield remote_server
19 +
20 +
21 +@pytest.mark.benchmark
22 +def test_bench_serve(server):
23 + command = [
24 + "vllm",
25 + "bench",
26 + "serve",
27 + "--model",
28 + MODEL_NAME,
29 + "--host",
30 + server.host,
31 + "--port",
32 + str(server.port),
33 + "--random-input-len",
34 + "32",
35 + "--random-output-len",
36 + "4",
37 + "--num-prompts",
38 + "5",
39 + ]
40 + result = subprocess.run(command, capture_output=True, text=True)
41 + print(result.stdout)
42 + print(result.stderr)
43 +
44 + assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
tests/benchmarks/test_throughput_cli.py new
+19
@@ -0,0 +1,19 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import subprocess
3 +
4 +import pytest
5 +
6 +MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
7 +
8 +
9 +@pytest.mark.benchmark
10 +def test_bench_throughput():
11 + command = [
12 + "vllm", "bench", "throughput", "--model", MODEL_NAME, "--input-len",
13 + "32", "--output-len", "1", "--enforce-eager", "--load-format", "dummy"
14 + ]
15 + result = subprocess.run(command, capture_output=True, text=True)
16 + print(result.stdout)
17 + print(result.stderr)
18 +
19 + assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
vllm/benchmarks/datasets.py new
+831
@@ -0,0 +1,831 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""
3 +This module defines a framework for sampling benchmark requests from various
4 +datasets. Each dataset subclass of BenchmarkDataset must implement sample
5 +generation. Supported dataset types include:
6 + - ShareGPT
7 + - Random (synthetic)
8 + - Sonnet
9 + - BurstGPT
10 + - HuggingFace
11 + - VisionArena
12 +
13 +TODO: Implement CustomDataset to parse a JSON file and convert its contents into
14 +SampleRequest instances, similar to the approach used in ShareGPT.
15 +"""
16 +
17 +import base64
18 +import io
19 +import json
20 +import logging
21 +import random
22 +from abc import ABC, abstractmethod
23 +from collections.abc import Mapping
24 +from dataclasses import dataclass
25 +from functools import cache
26 +from io import BytesIO
27 +from typing import Any, Callable, Optional, Union
28 +
29 +import numpy as np
30 +from PIL import Image
31 +from transformers import PreTrainedTokenizerBase
32 +
33 +from vllm.lora.request import LoRARequest
34 +from vllm.lora.utils import get_adapter_absolute_path
35 +from vllm.multimodal import MultiModalDataDict
36 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer
37 +
38 +logger = logging.getLogger(__name__)
39 +
40 +# -----------------------------------------------------------------------------
41 +# Data Classes
42 +# -----------------------------------------------------------------------------
43 +
44 +
45 +@dataclass
46 +class SampleRequest:
47 + """
48 + Represents a single inference request for benchmarking.
49 + """
50 +
51 + prompt: Union[str, Any]
52 + prompt_len: int
53 + expected_output_len: int
54 + multi_modal_data: Optional[Union[MultiModalDataDict, dict]] = None
55 + lora_request: Optional[LoRARequest] = None
56 +
57 +
58 +# -----------------------------------------------------------------------------
59 +# Benchmark Dataset Base Class
60 +# -----------------------------------------------------------------------------
61 +
62 +
63 +class BenchmarkDataset(ABC):
64 + DEFAULT_SEED = 0
65 +
66 + def __init__(
67 + self,
68 + dataset_path: Optional[str] = None,
69 + random_seed: int = DEFAULT_SEED,
70 + ) -> None:
71 + """
72 + Initialize the BenchmarkDataset with an optional dataset path and random
73 + seed.
74 +
75 + Args:
76 + dataset_path (Optional[str]): Path to the dataset. If None, it
77 + indicates that a default or random dataset might be used.
78 + random_seed (int): Seed value for reproducible shuffling or
79 + sampling. Defaults to DEFAULT_SEED.
80 + """
81 + self.dataset_path = dataset_path
82 + # Set the random seed, ensuring that a None value is replaced with the
83 + # default seed.
84 + self.random_seed = (random_seed
85 + if random_seed is not None else self.DEFAULT_SEED)
86 + self.data = None
87 +
88 + def apply_multimodal_chat_transformation(
89 + self,
90 + prompt: str,
91 + mm_content: Optional[MultiModalDataDict] = None) -> list[dict]:
92 + """
93 + Transform a prompt and optional multimodal content into a chat format.
94 + This method is used for chat models that expect a specific conversation
95 + format.
96 + """
97 + content = [{"text": prompt, "type": "text"}]
98 + if mm_content is not None:
99 + content.append(mm_content)
100 + return [{"role": "user", "content": content}]
101 +
102 + def load_data(self) -> None:
103 + """
104 + Load data from the dataset path into self.data.
105 +
106 + This method must be overridden by subclasses since the method to load
107 + data will vary depending on the dataset format and source.
108 +
109 + Raises:
110 + NotImplementedError: If a subclass does not implement this method.
111 + """
112 + # TODO (jenniferzhao): add support for downloading data
113 + raise NotImplementedError(
114 + "load_data must be implemented in subclasses.")
115 +
116 + def get_random_lora_request(
117 + self,
118 + tokenizer: PreTrainedTokenizerBase,
119 + max_loras: Optional[int] = None,
120 + lora_path: Optional[str] = None,
121 + ) -> tuple[Optional[LoRARequest], AnyTokenizer]:
122 + """
123 + Optionally select a random LoRA request and return its associated
124 + tokenizer.
125 +
126 + This method is used when LoRA parameters are provided. It randomly
127 + selects a LoRA based on max_loras and retrieves a cached tokenizer for
128 + that LoRA if available. Otherwise, it returns the base tokenizer.
129 +
130 + Args:
131 + tokenizer (PreTrainedTokenizerBase): The base tokenizer to use if no
132 + LoRA is selected. max_loras (Optional[int]): The maximum number of
133 + LoRAs available. If None, LoRA is not used. lora_path
134 + (Optional[str]): Path to the LoRA parameters on disk. If None, LoRA
135 + is not used.
136 +
137 + Returns:
138 + tuple[Optional[LoRARequest], AnyTokenizer]: A tuple where the first
139 + element is a LoRARequest (or None if not applicable) and the second
140 + element is the tokenizer associated with the LoRA request (or the
141 + base tokenizer).
142 + """
143 + if max_loras is None or lora_path is None:
144 + return None, tokenizer
145 +
146 + # Generate a random LoRA ID in the range [1, max_loras].
147 + lora_id = random.randint(1, max_loras)
148 + lora_request = LoRARequest(
149 + lora_name=str(lora_id),
150 + lora_int_id=lora_id,
151 + lora_path=lora_path_on_disk(lora_path),
152 + )
153 + if lora_id not in lora_tokenizer_cache:
154 + lora_tokenizer_cache[lora_id] = get_lora_tokenizer(lora_request)
155 + # Return lora_request and the cached tokenizer if available; otherwise,
156 + # return the base tokenizer
157 + return lora_request, lora_tokenizer_cache[lora_id] or tokenizer
158 +
159 + @abstractmethod
160 + def sample(self, tokenizer: PreTrainedTokenizerBase,
161 + num_requests: int) -> list[SampleRequest]:
162 + """
163 + Abstract method to generate sample requests from the dataset.
164 +
165 + Subclasses must override this method to implement dataset-specific logic
166 + for generating a list of SampleRequest objects.
167 +
168 + Args:
169 + tokenizer (PreTrainedTokenizerBase): The tokenizer to be used
170 + for processing the dataset's text.
171 + num_requests (int): The number of sample requests to generate.
172 +
173 + Returns:
174 + list[SampleRequest]: A list of sample requests generated from the
175 + dataset.
176 + """
177 + raise NotImplementedError("sample must be implemented in subclasses.")
178 +
179 + def maybe_oversample_requests(self, requests: list[SampleRequest],
180 + num_requests: int) -> None:
181 + """
182 + Oversamples the list of requests if its size is less than the desired
183 + number.
184 +
185 + Args:
186 + requests (List[SampleRequest]): The current list of sampled
187 + requests. num_requests (int): The target number of requests.
188 + """
189 + if len(requests) < num_requests:
190 + random.seed(self.random_seed)
191 + additional = random.choices(requests,
192 + k=num_requests - len(requests))
193 + requests.extend(additional)
194 + logger.info("Oversampled requests to reach %d total samples.",
195 + num_requests)
196 +
197 +
198 +# -----------------------------------------------------------------------------
199 +# Utility Functions and Global Caches
200 +# -----------------------------------------------------------------------------
201 +
202 +
203 +def is_valid_sequence(
204 + prompt_len: int,
205 + output_len: int,
206 + min_len: int = 4,
207 + max_prompt_len: int = 1024,
208 + max_total_len: int = 2048,
209 + skip_min_output_len_check: bool = False,
210 +) -> bool:
211 + """
212 + Validate a sequence based on prompt and output lengths.
213 +
214 + Default pruning criteria are copied from the original `sample_hf_requests`
215 + and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as
216 + from `sample_requests` in benchmark_throughput.py.
217 + """
218 + # Check for invalid conditions
219 + prompt_too_short = prompt_len < min_len
220 + output_too_short = (not skip_min_output_len_check) and (output_len
221 + < min_len)
222 + prompt_too_long = prompt_len > max_prompt_len
223 + combined_too_long = (prompt_len + output_len) > max_total_len
224 +
225 + # Return True if none of the invalid conditions are met
226 + return not (prompt_too_short or output_too_short or prompt_too_long
227 + or combined_too_long)
228 +
229 +
230 +@cache
231 +def lora_path_on_disk(lora_path: str) -> str:
232 + return get_adapter_absolute_path(lora_path)
233 +
234 +
235 +# Global cache for LoRA tokenizers.
236 +lora_tokenizer_cache: dict[int, AnyTokenizer] = {}
237 +
238 +
239 +def process_image(image: Any) -> Mapping[str, Any]:
240 + """
241 + Process a single image input and return a multimedia content dictionary.
242 +
243 + Supports three input types:
244 +
245 + 1. Dictionary with raw image bytes: - Expects a dict with a 'bytes' key
246 + containing raw image data. - Loads the bytes as a PIL.Image.Image.
247 +
248 + 2. PIL.Image.Image input: - Converts the image to RGB. - Saves the image as
249 + a JPEG in memory. - Encodes the JPEG data as a base64 string. - Returns
250 + a dictionary with the image as a base64 data URL.
251 +
252 + 3. String input: - Treats the string as a URL or local file path. -
253 + Prepends "file://" if the string doesn't start with "http://" or
254 + "file://". - Returns a dictionary with the image URL.
255 +
256 + Raises:
257 + ValueError: If the input is not a supported type.
258 + """
259 + if isinstance(image, dict) and 'bytes' in image:
260 + image = Image.open(BytesIO(image['bytes']))
261 + if isinstance(image, Image.Image):
262 + image = image.convert("RGB")
263 + with io.BytesIO() as image_data:
264 + image.save(image_data, format="JPEG")
265 + image_base64 = base64.b64encode(
266 + image_data.getvalue()).decode("utf-8")
267 + return {
268 + "type": "image_url",
269 + "image_url": {
270 + "url": f"data:image/jpeg;base64,{image_base64}"
271 + },
272 + }
273 +
274 + if isinstance(image, str):
275 + image_url = (image if image.startswith(
276 + ("http://", "file://")) else f"file://{image}")
277 + return {"type": "image_url", "image_url": {"url": image_url}}
278 +
279 + raise ValueError(f"Invalid image input {image}. Must be a PIL.Image.Image"
280 + " or str or dictionary with raw image bytes.")
281 +
282 +
283 +# -----------------------------------------------------------------------------
284 +# Random Dataset Implementation (Synthetic Data)
285 +# -----------------------------------------------------------------------------
286 +
287 +
288 +class RandomDataset(BenchmarkDataset):
289 + # Default values copied from benchmark_serving.py for the random dataset.
290 + DEFAULT_PREFIX_LEN = 0
291 + DEFAULT_RANGE_RATIO = 0.0
292 + DEFAULT_INPUT_LEN = 1024
293 + DEFAULT_OUTPUT_LEN = 128
294 +
295 + def __init__(
296 + self,
297 + **kwargs,
298 + ) -> None:
299 + super().__init__(**kwargs)
300 +
301 + def sample(
302 + self,
303 + tokenizer: PreTrainedTokenizerBase,
304 + num_requests: int,
305 + prefix_len: int = DEFAULT_PREFIX_LEN,
306 + range_ratio: float = DEFAULT_RANGE_RATIO,
307 + input_len: int = DEFAULT_INPUT_LEN,
308 + output_len: int = DEFAULT_OUTPUT_LEN,
309 + **kwargs,
310 + ) -> list[SampleRequest]:
311 + # Enforce range_ratio < 1
312 + assert range_ratio < 1.0, (
313 + "random_range_ratio must be < 1.0 to ensure a valid sampling range"
314 + )
315 +
316 + vocab_size = tokenizer.vocab_size
317 +
318 + prefix_token_ids = (np.random.randint(
319 + 0, vocab_size, size=prefix_len).tolist() if prefix_len > 0 else [])
320 +
321 + # New sampling logic: [X * (1 - b), X * (1 + b)]
322 + input_low = int(input_len * (1 - range_ratio))
323 + input_high = int(input_len * (1 + range_ratio))
324 + output_low = int(output_len * (1 - range_ratio))
325 + output_high = int(output_len * (1 + range_ratio))
326 +
327 + # Add logging for debugging
328 + logger.info("Sampling input_len from [%s, %s]", input_low, input_high)
329 + logger.info("Sampling output_len from [%s, %s]", output_low,
330 + output_high)
331 +
332 + input_lens = np.random.randint(input_low,
333 + input_high + 1,
334 + size=num_requests)
335 + output_lens = np.random.randint(output_low,
336 + output_high + 1,
337 + size=num_requests)
338 + offsets = np.random.randint(0, vocab_size, size=num_requests)
339 +
340 + requests = []
341 + for i in range(num_requests):
342 + inner_seq = ((offsets[i] + i + np.arange(input_lens[i])) %
343 + vocab_size).tolist()
344 + token_sequence = prefix_token_ids + inner_seq
345 + prompt = tokenizer.decode(token_sequence)
346 + total_input_len = prefix_len + int(input_lens[i])
347 + requests.append(
348 + SampleRequest(
349 + prompt=prompt,
350 + prompt_len=total_input_len,
351 + expected_output_len=int(output_lens[i]),
352 + ))
353 + return requests
354 +
355 +
356 +# -----------------------------------------------------------------------------
357 +# ShareGPT Dataset Implementation
358 +# -----------------------------------------------------------------------------
359 +
360 +
361 +class ShareGPTDataset(BenchmarkDataset):
362 + """
363 + Implements the ShareGPT dataset. Loads data from a JSON file and generates
364 + sample requests based on conversation turns.
365 + """
366 +
367 + def __init__(self, **kwargs) -> None:
368 + super().__init__(**kwargs)
369 + self.load_data()
370 +
371 + def load_data(self) -> None:
372 + if self.dataset_path is None:
373 + raise ValueError("dataset_path must be provided for loading data.")
374 +
375 + with open(self.dataset_path, encoding="utf-8") as f:
376 + self.data = json.load(f)
377 + # Filter entries with at least two conversation turns.
378 + self.data = [
379 + entry for entry in self.data
380 + if "conversations" in entry and len(entry["conversations"]) >= 2
381 + ]
382 + random.seed(self.random_seed)
383 + random.shuffle(self.data)
384 +
385 + def sample(
386 + self,
387 + tokenizer: PreTrainedTokenizerBase,
388 + num_requests: int,
389 + lora_path: Optional[str] = None,
390 + max_loras: Optional[int] = None,
391 + output_len: Optional[int] = None,
392 + enable_multimodal_chat: bool = False,
393 + **kwargs,
394 + ) -> list:
395 + samples: list = []
396 + for entry in self.data:
397 + if len(samples) >= num_requests:
398 + break
399 + prompt, completion = (
400 + entry["conversations"][0]["value"],
401 + entry["conversations"][1]["value"],
402 + )
403 +
404 + lora_request, tokenizer = self.get_random_lora_request(
405 + tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path)
406 + prompt_ids = tokenizer(prompt).input_ids
407 + completion_ids = tokenizer(completion).input_ids
408 + prompt_len = len(prompt_ids)
409 + new_output_len = (len(completion_ids)
410 + if output_len is None else output_len)
411 + if not is_valid_sequence(prompt_len,
412 + new_output_len,
413 + skip_min_output_len_check=output_len
414 + is not None):
415 + continue
416 + if enable_multimodal_chat:
417 + prompt = self.apply_multimodal_chat_transformation(
418 + prompt, None)
419 + samples.append(
420 + SampleRequest(
421 + prompt=prompt,
422 + prompt_len=prompt_len,
423 + expected_output_len=new_output_len,
424 + lora_request=lora_request,
425 + ))
426 + self.maybe_oversample_requests(samples, num_requests)
427 + return samples
428 +
429 +
430 +# -----------------------------------------------------------------------------
431 +# Sonnet Dataset Implementation
432 +# -----------------------------------------------------------------------------
433 +
434 +
435 +class SonnetDataset(BenchmarkDataset):
436 + """
437 + Simplified implementation of the Sonnet dataset. Loads poem lines from a
438 + text file and generates sample requests. Default values here copied from
439 + `benchmark_serving.py` for the sonnet dataset.
440 + """
441 +
442 + DEFAULT_PREFIX_LEN = 200
443 + DEFAULT_INPUT_LEN = 550
444 + DEFAULT_OUTPUT_LEN = 150
445 +
446 + def __init__(
447 + self,
448 + **kwargs,
449 + ) -> None:
450 + super().__init__(**kwargs)
451 + self.load_data()
452 +
453 + def load_data(self) -> None:
454 + if not self.dataset_path:
455 + raise ValueError("dataset_path must be provided.")
456 + with open(self.dataset_path, encoding="utf-8") as f:
457 + self.data = f.readlines()
458 +
459 + def sample(
460 + self,
461 + tokenizer,
462 + num_requests: int,
463 + prefix_len: int = DEFAULT_PREFIX_LEN,
464 + input_len: int = DEFAULT_INPUT_LEN,
465 + output_len: int = DEFAULT_OUTPUT_LEN,
466 + return_prompt_formatted: bool = False,
467 + **kwargs,
468 + ) -> list:
469 + # Calculate average token length for a poem line.
470 + tokenized_lines = [tokenizer(line).input_ids for line in self.data]
471 + avg_len = sum(len(tokens)
472 + for tokens in tokenized_lines) / len(tokenized_lines)
473 +
474 + # Build the base prompt.
475 + base_prompt = "Pick as many lines as you can from these poem lines:\n"
476 + base_msg = [{"role": "user", "content": base_prompt}]
477 + base_fmt = tokenizer.apply_chat_template(base_msg,
478 + add_generation_prompt=True,
479 + tokenize=False)
480 + base_offset = len(tokenizer(base_fmt).input_ids)
481 + if input_len <= base_offset:
482 + raise ValueError(
483 + f"'input_len' must be higher than the base prompt length "
484 + f"({base_offset}).")
485 +
486 + # Determine how many poem lines to use.
487 + num_input_lines = round((input_len - base_offset) / avg_len)
488 + num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
489 + prefix_lines = self.data[:num_prefix_lines]
490 +
491 + samples = []
492 + while len(samples) < num_requests:
493 + extra_lines = random.choices(self.data,
494 + k=num_input_lines - num_prefix_lines)
495 + prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
496 + msg = [{"role": "user", "content": prompt}]
497 + prompt_formatted = tokenizer.apply_chat_template(
498 + msg, add_generation_prompt=True, tokenize=False)
499 + prompt_len = len(tokenizer(prompt_formatted).input_ids)
500 + if prompt_len <= input_len:
501 + samples.append(
502 + SampleRequest(
503 + prompt=prompt_formatted
504 + if return_prompt_formatted else prompt,
505 + prompt_len=prompt_len,
506 + expected_output_len=output_len,
507 + ))
508 + return samples
509 +
510 +
511 +# -----------------------------------------------------------------------------
512 +# BurstGPT Dataset Implementation
513 +# -----------------------------------------------------------------------------
514 +
515 +
516 +class BurstGPTDataset(BenchmarkDataset):
517 + """
518 + Implements the BurstGPT dataset. Loads data from a CSV file and generates
519 + sample requests based on synthetic prompt generation. Only rows with Model
520 + "GPT-4" and positive response tokens are used.
521 + """
522 +
523 + def __init__(self, **kwargs) -> None:
524 + super().__init__(**kwargs)
525 + self.load_data()
526 +
527 + def load_data(self, ):
528 + if self.dataset_path is None:
529 + raise ValueError("dataset_path must be provided for loading data.")
530 +
531 + try:
532 + import pandas as pd
533 + except ImportError as e:
534 + raise ImportError(
535 + "Pandas is required for BurstGPTDataset. Please install it "
536 + "using `pip install pandas`.") from e
537 +
538 + df = pd.read_csv(self.dataset_path)
539 + # Filter to keep only GPT-4 rows.
540 + gpt4_df = df[df["Model"] == "GPT-4"]
541 + # Remove failed requests (where Response tokens is 0 or less).
542 + gpt4_df = gpt4_df[gpt4_df["Response tokens"] > 0]
543 + # Sample the desired number of rows.
544 + self.data = gpt4_df
545 +
546 + def _sample_loaded_data(self, num_requests: int) -> list:
547 + if num_requests <= len(self.data):
548 + data = self.data.sample(n=num_requests,
549 + random_state=self.random_seed)
550 + else:
551 + data = self.data.sample(
552 + n=num_requests,
553 + random_state=self.random_seed,
554 + replace=True,
555 + )
556 + # Convert the dataframe to a list of lists.
557 + return data.values.tolist()
558 +
559 + def sample(
560 + self,
561 + tokenizer: PreTrainedTokenizerBase,
562 + num_requests: int,
563 + max_loras: Optional[int] = None,
564 + lora_path: Optional[str] = None,
565 + **kwargs,
566 + ) -> list[SampleRequest]:
567 + samples = []
568 + data = self._sample_loaded_data(num_requests=num_requests)
569 + for i in range(num_requests):
570 + input_len = int(data[i][2])
571 + output_len = int(data[i][3])
572 + lora_req, tokenizer = self.get_random_lora_request(
573 + tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path)
574 + vocab_size = tokenizer.vocab_size
575 + # Generate a synthetic prompt: a list of token IDs computed as (i +
576 + # j) modulo vocab_size.
577 + token_ids = [(i + j) % vocab_size for j in range(input_len)]
578 + prompt = tokenizer.decode(token_ids)
579 + samples.append(
580 + SampleRequest(
581 + prompt=prompt,
582 + prompt_len=input_len,
583 + expected_output_len=output_len,
584 + lora_request=lora_req,
585 + ))
586 + return samples
587 +
588 +
589 +# -----------------------------------------------------------------------------
590 +# HuggingFace Dataset Base Implementation
591 +# -----------------------------------------------------------------------------
592 +class HuggingFaceDataset(BenchmarkDataset):
593 + """Base class for datasets hosted on HuggingFace."""
594 +
595 + SUPPORTED_DATASET_PATHS: Union[set[str], dict[str, Callable]] = set()
596 +
597 + def __init__(
598 + self,
599 + dataset_path: str,
600 + dataset_split: str,
601 + dataset_subset: Optional[str] = None,
602 + **kwargs,
603 + ) -> None:
604 + super().__init__(dataset_path=dataset_path, **kwargs)
605 +
606 + self.dataset_split = dataset_split
607 + self.dataset_subset = dataset_subset
608 + self.load_data()
609 +
610 + def load_data(self) -> None:
611 + """Load data from HuggingFace datasets."""
612 + try:
613 + from datasets import load_dataset
614 + except ImportError as e:
615 + raise ImportError(
616 + "Hugging Face datasets library is required for this dataset. "
617 + "Please install it using `pip install datasets`.") from e
618 +
619 + self.data = load_dataset(
620 + self.dataset_path,
621 + name=self.dataset_subset,
622 + split=self.dataset_split,
623 + streaming=True,
624 + )
625 + self.data = self.data.shuffle(seed=self.random_seed)
626 +
627 +
628 +# -----------------------------------------------------------------------------
629 +# Conversation Dataset Implementation
630 +# -----------------------------------------------------------------------------
631 +
632 +
633 +class ConversationDataset(HuggingFaceDataset):
634 + """Dataset for conversation data with multimodal support."""
635 + SUPPORTED_DATASET_PATHS = {
636 + 'lmms-lab/LLaVA-OneVision-Data', 'Aeala/ShareGPT_Vicuna_unfiltered'
637 + }
638 +
639 + def sample(self,
640 + tokenizer: PreTrainedTokenizerBase,
641 + num_requests: int,
642 + output_len: Optional[int] = None,
643 + enable_multimodal_chat: bool = False,
644 + **kwargs) -> list:
645 + # Filter examples with at least 2 conversations
646 + filtered_data = self.data.filter(
647 + lambda x: len(x["conversations"]) >= 2)
648 + sampled_requests = []
649 + dynamic_output = output_len is None
650 +
651 + for item in filtered_data:
652 + if len(sampled_requests) >= num_requests:
653 + break
654 + conv = item["conversations"]
655 + prompt, completion = conv[0]["value"], conv[1]["value"]
656 +
657 + prompt_ids = tokenizer(prompt).input_ids
658 + completion_ids = tokenizer(completion).input_ids
659 + prompt_len = len(prompt_ids)
660 + completion_len = len(completion_ids)
661 + output_len = completion_len if dynamic_output else output_len
662 + assert isinstance(output_len, int) and output_len > 0
663 + if dynamic_output and not is_valid_sequence(
664 + prompt_len, completion_len):
665 + continue
666 + mm_content = process_image(
667 + item["image"]) if "image" in item else None
668 + if enable_multimodal_chat:
669 + # Note: when chat is enabled the request prompt_len is no longer
670 + # accurate and we will be using request output to count the
671 + # actual prompt len and output len
672 + prompt = self.apply_multimodal_chat_transformation(
673 + prompt, mm_content)
674 + sampled_requests.append(
675 + SampleRequest(
676 + prompt=prompt,
677 + prompt_len=prompt_len,
678 + expected_output_len=output_len,
679 + multi_modal_data=mm_content,
680 + ))
681 + self.maybe_oversample_requests(sampled_requests, num_requests)
682 + return sampled_requests
683 +
684 +
685 +# -----------------------------------------------------------------------------
686 +# Vision Arena Dataset Implementation
687 +# -----------------------------------------------------------------------------
688 +
689 +
690 +class VisionArenaDataset(HuggingFaceDataset):
691 + """
692 + Vision Arena Dataset.
693 + """
694 +
695 + DEFAULT_OUTPUT_LEN = 128
696 + SUPPORTED_DATASET_PATHS = {
697 + "lmarena-ai/VisionArena-Chat":
698 + lambda x: x["conversation"][0][0]["content"],
699 + "lmarena-ai/vision-arena-bench-v0.1":
700 + lambda x: x["turns"][0][0]["content"]
701 + }
702 +
703 + def sample(
704 + self,
705 + tokenizer: PreTrainedTokenizerBase,
706 + num_requests: int,
707 + output_len: Optional[int] = None,
708 + enable_multimodal_chat: bool = False,
709 + **kwargs,
710 + ) -> list:
711 + output_len = (output_len
712 + if output_len is not None else self.DEFAULT_OUTPUT_LEN)
713 + sampled_requests = []
714 + for item in self.data:
715 + if len(sampled_requests) >= num_requests:
716 + break
717 + parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.dataset_path)
718 + if parser_fn is None:
719 + raise ValueError(
720 + f"Unsupported dataset path: {self.dataset_path}")
721 + prompt = parser_fn(item)
722 + mm_content = process_image(item["images"][0])
723 + prompt_len = len(tokenizer(prompt).input_ids)
724 + if enable_multimodal_chat:
725 + # Note: when chat is enabled the request prompt_len is no longer
726 + # accurate and we will be using request output to count the
727 + # actual prompt len
728 + prompt = self.apply_multimodal_chat_transformation(
729 + prompt, mm_content)
730 + sampled_requests.append(
731 + SampleRequest(
732 + prompt=prompt,
733 + prompt_len=prompt_len,
734 + expected_output_len=output_len,
735 + multi_modal_data=mm_content,
736 + ))
737 + self.maybe_oversample_requests(sampled_requests, num_requests)
738 + return sampled_requests
739 +
740 +
741 +# -----------------------------------------------------------------------------
742 +# Instruct Coder Dataset Implementation
743 +# -----------------------------------------------------------------------------
744 +
745 +
746 +class InstructCoderDataset(HuggingFaceDataset):
747 + """
748 + InstructCoder Dataset.
749 + https://huggingface.co/datasets/likaixin/InstructCoder
750 +
751 + InstructCoder is the dataset designed for general code editing. It consists
752 + of 114,239 instruction-input-output triplets, and covers multiple distinct
753 + code editing scenario.
754 + """
755 +
756 + DEFAULT_OUTPUT_LEN = 200 # this is the average default output length
757 + SUPPORTED_DATASET_PATHS = {
758 + "likaixin/InstructCoder",
759 + }
760 +
761 + def sample(self,
762 + tokenizer: PreTrainedTokenizerBase,
763 + num_requests: int,
764 + output_len: Optional[int] = None,
765 + enable_multimodal_chat: bool = False,
766 + **kwargs) -> list:
767 + output_len = (output_len
768 + if output_len is not None else self.DEFAULT_OUTPUT_LEN)
769 + sampled_requests = []
770 + for item in self.data:
771 + if len(sampled_requests) >= num_requests:
772 + break
773 + prompt = f"{item['instruction']}:\n{item['input']}"
774 + prompt_len = len(tokenizer(prompt).input_ids)
775 + sampled_requests.append(
776 + SampleRequest(
777 + prompt=prompt,
778 + prompt_len=prompt_len,
779 + expected_output_len=output_len,
780 + ))
781 + self.maybe_oversample_requests(sampled_requests, num_requests)
782 + return sampled_requests
783 +
784 +
785 +# -----------------------------------------------------------------------------
786 +# AIMO Dataset Implementation
787 +# -----------------------------------------------------------------------------
788 +
789 +
790 +class AIMODataset(HuggingFaceDataset):
791 + """
792 + Dataset class for processing a AIMO dataset with reasoning questions.
793 + """
794 + SUPPORTED_DATASET_PATHS = {
795 + "AI-MO/aimo-validation-aime", "AI-MO/NuminaMath-1.5",
796 + "AI-MO/NuminaMath-CoT"
797 + }
798 +
799 + def sample(self,
800 + tokenizer: PreTrainedTokenizerBase,
801 + num_requests: int,
802 + output_len: Optional[int] = None,
803 + **kwargs) -> list:
804 + sampled_requests = []
805 + dynamic_output = output_len is None
806 +
807 + for item in self.data:
808 + if len(sampled_requests) >= num_requests:
809 + break
810 + prompt, completion = item['problem'], item["solution"]
811 +
812 + prompt_ids = tokenizer(prompt).input_ids
813 + completion_ids = tokenizer(completion).input_ids
814 + prompt_len = len(prompt_ids)
815 + completion_len = len(completion_ids)
816 + output_len = completion_len if dynamic_output else output_len
817 + assert isinstance(output_len, int) and output_len > 0
818 + if dynamic_output and not is_valid_sequence(prompt_len,
819 + completion_len,
820 + max_prompt_len=2048,
821 + max_total_len=32000):
822 + continue
823 + sampled_requests.append(
824 + SampleRequest(
825 + prompt=prompt,
826 + prompt_len=prompt_len,
827 + expected_output_len=output_len,
828 + multi_modal_data=None,
829 + ))
830 + self.maybe_oversample_requests(sampled_requests, num_requests)
831 + return sampled_requests
vllm/benchmarks/latency.py new
+181
@@ -0,0 +1,181 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""Benchmark the latency of processing a single batch of requests."""
3 +
4 +import argparse
5 +import dataclasses
6 +import json
7 +import os
8 +import time
9 +from pathlib import Path
10 +from typing import Any, Optional
11 +
12 +import numpy as np
13 +import torch
14 +from tqdm import tqdm
15 +
16 +from vllm import LLM, SamplingParams
17 +from vllm.benchmarks.utils import (convert_to_pytorch_benchmark_format,
18 + write_to_json)
19 +from vllm.engine.arg_utils import EngineArgs
20 +from vllm.inputs import PromptType
21 +from vllm.sampling_params import BeamSearchParams
22 +
23 +
24 +def save_to_pytorch_benchmark_format(args: argparse.Namespace,
25 + results: dict[str, Any]) -> None:
26 + pt_records = convert_to_pytorch_benchmark_format(
27 + args=args,
28 + metrics={"latency": results["latencies"]},
29 + extra_info={k: results[k]
30 + for k in ["avg_latency", "percentiles"]})
31 + if pt_records:
32 + pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"
33 + write_to_json(pt_file, pt_records)
34 +
35 +
36 +def add_cli_args(parser: argparse.ArgumentParser):
37 + parser.add_argument("--input-len", type=int, default=32)
38 + parser.add_argument("--output-len", type=int, default=128)
39 + parser.add_argument("--batch-size", type=int, default=8)
40 + parser.add_argument(
41 + "--n",
42 + type=int,
43 + default=1,
44 + help="Number of generated sequences per prompt.",
45 + )
46 + parser.add_argument("--use-beam-search", action="store_true")
47 + parser.add_argument(
48 + "--num-iters-warmup",
49 + type=int,
50 + default=10,
51 + help="Number of iterations to run for warmup.",
52 + )
53 + parser.add_argument("--num-iters",
54 + type=int,
55 + default=30,
56 + help="Number of iterations to run.")
57 + parser.add_argument(
58 + "--profile",
59 + action="store_true",
60 + help="profile the generation process of a single batch",
61 + )
62 + parser.add_argument(
63 + "--profile-result-dir",
64 + type=str,
65 + default=None,
66 + help=("path to save the pytorch profiler output. Can be visualized "
67 + "with ui.perfetto.dev or Tensorboard."),
68 + )
69 + parser.add_argument(
70 + "--output-json",
71 + type=str,
72 + default=None,
73 + help="Path to save the latency results in JSON format.",
74 + )
75 + parser.add_argument(
76 + "--disable-detokenize",
77 + action="store_true",
78 + help=("Do not detokenize responses (i.e. do not include "
79 + "detokenization time in the latency measurement)"),
80 + )
81 +
82 + parser = EngineArgs.add_cli_args(parser)
83 +
84 +
85 +def main(args: argparse.Namespace):
86 + print(args)
87 +
88 + engine_args = EngineArgs.from_cli_args(args)
89 +
90 + # NOTE(woosuk): If the request cannot be processed in a single batch,
91 + # the engine will automatically process the request in multiple batches.
92 + llm = LLM(**dataclasses.asdict(engine_args))
93 + assert llm.llm_engine.model_config.max_model_len >= (
94 + args.input_len +
95 + args.output_len), ("Please ensure that max_model_len is greater than"
96 + " the sum of input_len and output_len.")
97 +
98 + sampling_params = SamplingParams(
99 + n=args.n,
100 + temperature=1.0,
101 + top_p=1.0,
102 + ignore_eos=True,
103 + max_tokens=args.output_len,
104 + detokenize=not args.disable_detokenize,
105 + )
106 + print(sampling_params)
107 + dummy_prompt_token_ids = np.random.randint(10000,
108 + size=(args.batch_size,
109 + args.input_len))
110 + dummy_prompts: list[PromptType] = [{
111 + "prompt_token_ids": batch
112 + } for batch in dummy_prompt_token_ids.tolist()]
113 +
114 + def llm_generate():
115 + if not args.use_beam_search:
116 + llm.generate(dummy_prompts,
117 + sampling_params=sampling_params,
118 + use_tqdm=False)
119 + else:
120 + llm.beam_search(
121 + dummy_prompts,
122 + BeamSearchParams(
123 + beam_width=args.n,
124 + max_tokens=args.output_len,
125 + ignore_eos=True,
126 + ),
127 + )
128 +
129 + def run_to_completion(profile_dir: Optional[str] = None):
130 + if profile_dir:
131 + with torch.profiler.profile(
132 + activities=[
133 + torch.profiler.ProfilerActivity.CPU,
134 + torch.profiler.ProfilerActivity.CUDA,
135 + ],
136 + on_trace_ready=torch.profiler.tensorboard_trace_handler(
137 + str(profile_dir)),
138 + ) as p:
139 + llm_generate()
140 + print(p.key_averages().table(sort_by="self_cuda_time_total"))
141 + else:
142 + start_time = time.perf_counter()
143 + llm_generate()
144 + end_time = time.perf_counter()
145 + latency = end_time - start_time
146 + return latency
147 +
148 + print("Warming up...")
149 + for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
150 + run_to_completion(profile_dir=None)
151 +
152 + if args.profile:
153 + profile_dir = args.profile_result_dir
154 + if not profile_dir:
155 + profile_dir = (Path(".") / "vllm_benchmark_result" /
156 + f"latency_result_{time.time()}")
157 + print(f"Profiling (results will be saved to '{profile_dir}')...")
158 + run_to_completion(profile_dir=profile_dir)
159 + return
160 +
161 + # Benchmark.
162 + latencies = []
163 + for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
164 + latencies.append(run_to_completion(profile_dir=None))
165 + latencies = np.array(latencies)
166 + percentages = [10, 25, 50, 75, 90, 99]
167 + percentiles = np.percentile(latencies, percentages)
168 + print(f"Avg latency: {np.mean(latencies)} seconds")
169 + for percentage, percentile in zip(percentages, percentiles):
170 + print(f"{percentage}% percentile latency: {percentile} seconds")
171 +
172 + # Output JSON results if specified
173 + if args.output_json:
174 + results = {
175 + "avg_latency": np.mean(latencies),
176 + "latencies": latencies.tolist(),
177 + "percentiles": dict(zip(percentages, percentiles.tolist())),
178 + }
179 + with open(args.output_json, "w") as f:
180 + json.dump(results, f, indent=4)
181 + save_to_pytorch_benchmark_format(args, results)
vllm/benchmarks/throughput.py new
+608
@@ -0,0 +1,608 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""Benchmark offline inference throughput."""
3 +import argparse
4 +import dataclasses
5 +import json
6 +import os
7 +import random
8 +import time
9 +import warnings
10 +from typing import Any, Optional, Union
11 +
12 +import torch
13 +import uvloop
14 +from tqdm import tqdm
15 +from transformers import (AutoModelForCausalLM, AutoTokenizer,
16 + PreTrainedTokenizerBase)
17 +
18 +from vllm.benchmarks.datasets import (AIMODataset, BurstGPTDataset,
19 + ConversationDataset,
20 + InstructCoderDataset, RandomDataset,
21 + SampleRequest, ShareGPTDataset,
22 + SonnetDataset, VisionArenaDataset)
23 +from vllm.benchmarks.utils import (convert_to_pytorch_benchmark_format,
24 + write_to_json)
25 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
26 +from vllm.entrypoints.openai.api_server import (
27 + build_async_engine_client_from_engine_args)
28 +from vllm.inputs import TextPrompt, TokensPrompt
29 +from vllm.lora.request import LoRARequest
30 +from vllm.outputs import RequestOutput
31 +from vllm.sampling_params import BeamSearchParams
32 +from vllm.utils import merge_async_iterators
33 +
34 +
35 +def run_vllm(
36 + requests: list[SampleRequest],
37 + n: int,
38 + engine_args: EngineArgs,
39 + disable_detokenize: bool = False,
40 +) -> tuple[float, Optional[list[RequestOutput]]]:
41 + from vllm import LLM, SamplingParams
42 + llm = LLM(**dataclasses.asdict(engine_args))
43 + assert all(
44 + llm.llm_engine.model_config.max_model_len >= (
45 + request.prompt_len + request.expected_output_len)
46 + for request in requests), (
47 + "Please ensure that max_model_len is greater than the sum of"
48 + " prompt_len and expected_output_len for all requests.")
49 + # Add the requests to the engine.
50 + prompts: list[Union[TextPrompt, TokensPrompt]] = []
51 + sampling_params: list[SamplingParams] = []
52 + for request in requests:
53 + prompts.append(
54 + TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"],
55 + multi_modal_data=request.multi_modal_data)
56 + if "prompt_token_ids" in request.prompt else \
57 + TextPrompt(prompt=request.prompt,
58 + multi_modal_data=request.multi_modal_data))
59 + sampling_params.append(
60 + SamplingParams(
61 + n=n,
62 + temperature=1.0,
63 + top_p=1.0,
64 + ignore_eos=True,
65 + max_tokens=request.expected_output_len,
66 + detokenize=not disable_detokenize,
67 + ))
68 + lora_requests: Optional[list[LoRARequest]] = None
69 + if engine_args.enable_lora:
70 + lora_requests = [request.lora_request for request in requests]
71 +
72 + use_beam_search = False
73 +
74 + outputs = None
75 + if not use_beam_search:
76 + start = time.perf_counter()
77 + outputs = llm.generate(prompts,
78 + sampling_params,
79 + lora_request=lora_requests,
80 + use_tqdm=True)
81 + end = time.perf_counter()
82 + else:
83 + assert lora_requests is None, "BeamSearch API does not support LoRA"
84 + prompts = [request.prompt for request in requests]
85 + # output_len should be the same for all requests.
86 + output_len = requests[0][2]
87 + for request in requests:
88 + assert request.expected_output_len == output_len
89 + start = time.perf_counter()
90 + llm.beam_search(
91 + prompts,
92 + BeamSearchParams(
93 + beam_width=n,
94 + max_tokens=output_len,
95 + ignore_eos=True,
96 + ))
97 + end = time.perf_counter()
98 + return end - start, outputs
99 +
100 +
101 +def run_vllm_chat(
102 + requests: list[SampleRequest],
103 + n: int,
104 + engine_args: EngineArgs,
105 + disable_detokenize: bool = False) -> tuple[float, list[RequestOutput]]:
106 + """
107 + Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
108 + multimodal models as it properly handles multimodal inputs and chat
109 + formatting. For non-multimodal models, use run_vllm() instead.
110 + """
111 + from vllm import LLM, SamplingParams
112 + llm = LLM(**dataclasses.asdict(engine_args))
113 +
114 + assert all(
115 + llm.llm_engine.model_config.max_model_len >= (
116 + request.prompt_len + request.expected_output_len)
117 + for request in requests), (
118 + "Please ensure that max_model_len is greater than the sum of "
119 + "prompt_len and expected_output_len for all requests.")
120 +
121 + prompts = []
122 + sampling_params: list[SamplingParams] = []
123 + for request in requests:
124 + prompts.append(request.prompt)
125 + sampling_params.append(
126 + SamplingParams(
127 + n=n,
128 + temperature=1.0,
129 + top_p=1.0,
130 + ignore_eos=True,
131 + max_tokens=request.expected_output_len,
132 + detokenize=not disable_detokenize,
133 + ))
134 + start = time.perf_counter()
135 + outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
136 + end = time.perf_counter()
137 + return end - start, outputs
138 +
139 +
140 +async def run_vllm_async(
141 + requests: list[SampleRequest],
142 + n: int,
143 + engine_args: AsyncEngineArgs,
144 + disable_frontend_multiprocessing: bool = False,
145 + disable_detokenize: bool = False,
146 +) -> float:
147 + from vllm import SamplingParams
148 +
149 + async with build_async_engine_client_from_engine_args(
150 + engine_args, disable_frontend_multiprocessing) as llm:
151 + assert all(
152 + llm.model_config.max_model_len >= (request.prompt_len +
153 + request.expected_output_len)
154 + for request in requests), (
155 + "Please ensure that max_model_len is greater than the sum of"
156 + " prompt_len and expected_output_len for all requests.")
157 +
158 + # Add the requests to the engine.
159 + prompts: list[Union[TextPrompt, TokensPrompt]] = []
160 + sampling_params: list[SamplingParams] = []
161 + lora_requests: list[Optional[LoRARequest]] = []
162 + for request in requests:
163 + prompts.append(
164 + TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"],
165 + multi_modal_data=request.multi_modal_data)
166 + if "prompt_token_ids" in request.prompt else \
167 + TextPrompt(prompt=request.prompt,
168 + multi_modal_data=request.multi_modal_data))
169 + sampling_params.append(
170 + SamplingParams(
171 + n=n,
172 + temperature=1.0,
173 + top_p=1.0,
174 + ignore_eos=True,
175 + max_tokens=request.expected_output_len,
176 + detokenize=not disable_detokenize,
177 + ))
178 + lora_requests.append(request.lora_request)
179 +
180 + generators = []
181 + start = time.perf_counter()
182 + for i, (prompt, sp,
183 + lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
184 + generator = llm.generate(prompt,
185 + sp,
186 + lora_request=lr,
187 + request_id=f"test{i}")
188 + generators.append(generator)
189 + all_gens = merge_async_iterators(*generators)
190 + async for i, res in all_gens:
191 + pass
192 + end = time.perf_counter()
193 + return end - start
194 +
195 +
196 +def run_hf(
197 + requests: list[SampleRequest],
198 + model: str,
199 + tokenizer: PreTrainedTokenizerBase,
200 + n: int,
201 + max_batch_size: int,
202 + trust_remote_code: bool,
203 + disable_detokenize: bool = False,
204 +) -> float:
205 + llm = AutoModelForCausalLM.from_pretrained(
206 + model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
207 + if llm.config.model_type == "llama":
208 + # To enable padding in the HF backend.
209 + tokenizer.pad_token = tokenizer.eos_token
210 + llm = llm.cuda()
211 +
212 + pbar = tqdm(total=len(requests))
213 + start = time.perf_counter()
214 + batch: list[str] = []
215 + max_prompt_len = 0
216 + max_output_len = 0
217 + for i in range(len(requests)):
218 + prompt = requests[i].prompt
219 + prompt_len = requests[i].prompt_len
220 + output_len = requests[i].expected_output_len
221 + # Add the prompt to the batch.
222 + batch.append(prompt)
223 + max_prompt_len = max(max_prompt_len, prompt_len)
224 + max_output_len = max(max_output_len, output_len)
225 + if len(batch) < max_batch_size and i != len(requests) - 1:
226 + # Check if we can add more requests to the batch.
227 + next_prompt_len = requests[i + 1].prompt_len
228 + next_output_len = requests[i + 1].expected_output_len
229 + if (max(max_prompt_len, next_prompt_len) +
230 + max(max_output_len, next_output_len)) <= 2048:
231 + # We can add more requests to the batch.
232 + continue
233 +
234 + # Generate the sequences.
235 + input_ids = tokenizer(batch, return_tensors="pt",
236 + padding=True).input_ids
237 + llm_outputs = llm.generate(
238 + input_ids=input_ids.cuda(),
239 + do_sample=True,
240 + num_return_sequences=n,
241 + temperature=1.0,
242 + top_p=1.0,
243 + use_cache=True,
244 + max_new_tokens=max_output_len,
245 + )
246 + if not disable_detokenize:
247 + # Include the decoding time.
248 + tokenizer.batch_decode(llm_outputs, skip_special_tokens=True)
249 + pbar.update(len(batch))
250 +
251 + # Clear the batch.
252 + batch = []
253 + max_prompt_len = 0
254 + max_output_len = 0
255 + end = time.perf_counter()
256 + return end - start
257 +
258 +
259 +def save_to_pytorch_benchmark_format(args: argparse.Namespace,
260 + results: dict[str, Any]) -> None:
261 + pt_records = convert_to_pytorch_benchmark_format(
262 + args=args,
263 + metrics={
264 + "requests_per_second": [results["requests_per_second"]],
265 + "tokens_per_second": [results["tokens_per_second"]],
266 + },
267 + extra_info={
268 + k: results[k]
269 + for k in ["elapsed_time", "num_requests", "total_num_tokens"]
270 + })
271 + if pt_records:
272 + # Don't use json suffix here as we don't want CI to pick it up
273 + pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"
274 + write_to_json(pt_file, pt_records)
275 +
276 +
277 +def get_requests(args, tokenizer):
278 + # Common parameters for all dataset types.
279 + common_kwargs = {
280 + "dataset_path": args.dataset_path,
281 + "random_seed": args.seed,
282 + }
283 + sample_kwargs = {
284 + "tokenizer": tokenizer,
285 + "lora_path": args.lora_path,
286 + "max_loras": args.max_loras,
287 + "num_requests": args.num_prompts,
288 + "input_len": args.input_len,
289 + "output_len": args.output_len,
290 + }
291 +
292 + if args.dataset_path is None or args.dataset_name == "random":
293 + sample_kwargs["range_ratio"] = args.random_range_ratio
294 + sample_kwargs["prefix_len"] = args.prefix_len
295 + dataset_cls = RandomDataset
296 + elif args.dataset_name == "sharegpt":
297 + dataset_cls = ShareGPTDataset
298 + if args.backend == "vllm-chat":
299 + sample_kwargs["enable_multimodal_chat"] = True
300 + elif args.dataset_name == "sonnet":
301 + assert tokenizer.chat_template or tokenizer.default_chat_template, (
302 + "Tokenizer/model must have chat template for sonnet dataset.")
303 + dataset_cls = SonnetDataset
304 + sample_kwargs["prefix_len"] = args.prefix_len
305 + sample_kwargs["return_prompt_formatted"] = True
306 + elif args.dataset_name == "burstgpt":
307 + dataset_cls = BurstGPTDataset
308 + elif args.dataset_name == "hf":
309 + if args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS:
310 + dataset_cls = VisionArenaDataset
311 + common_kwargs['dataset_subset'] = None
312 + common_kwargs['dataset_split'] = "train"
313 + sample_kwargs["enable_multimodal_chat"] = True
314 + elif args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS:
315 + dataset_cls = InstructCoderDataset
316 + common_kwargs['dataset_split'] = "train"
317 + elif args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS:
318 + dataset_cls = ConversationDataset
319 + common_kwargs['dataset_subset'] = args.hf_subset
320 + common_kwargs['dataset_split'] = args.hf_split
321 + sample_kwargs["enable_multimodal_chat"] = True
322 + elif args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS:
323 + dataset_cls = AIMODataset
324 + common_kwargs['dataset_subset'] = None
325 + common_kwargs['dataset_split'] = "train"
326 + else:
327 + raise ValueError(f"Unknown dataset name: {args.dataset_name}")
328 + # Remove None values
329 + sample_kwargs = {k: v for k, v in sample_kwargs.items() if v is not None}
330 + return dataset_cls(**common_kwargs).sample(**sample_kwargs)
331 +
332 +
333 +def validate_args(args):
334 + """
335 + Validate command-line arguments.
336 + """
337 +
338 + # === Deprecation and Defaulting ===
339 + if args.dataset is not None:
340 + warnings.warn(
341 + "The '--dataset' argument will be deprecated in the next release. "
342 + "Please use '--dataset-name' and '--dataset-path' instead.",
343 + stacklevel=2)
344 + args.dataset_path = args.dataset
345 +
346 + if not getattr(args, "tokenizer", None):
347 + args.tokenizer = args.model
348 +
349 + # === Backend Validation ===
350 + valid_backends = {"vllm", "hf", "mii", "vllm-chat"}
351 + if args.backend not in valid_backends:
352 + raise ValueError(f"Unsupported backend: {args.backend}")
353 +
354 + # === Dataset Configuration ===
355 + if not args.dataset and not args.dataset_path:
356 + print(
357 + "When dataset path is not set, it will default to random dataset")
358 + args.dataset_name = 'random'
359 + if args.input_len is None:
360 + raise ValueError("input_len must be provided for a random dataset")
361 +
362 + # === Dataset Name Specific Checks ===
363 + # --hf-subset and --hf-split: only used
364 + # when dataset_name is 'hf'
365 + if args.dataset_name != "hf" and (
366 + getattr(args, "hf_subset", None) is not None
367 + or getattr(args, "hf_split", None) is not None):
368 + warnings.warn("--hf-subset and --hf-split will be ignored \
369 + since --dataset-name is not 'hf'.",
370 + stacklevel=2)
371 + elif args.dataset_name == "hf":
372 + if args.dataset_path in (
373 + VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
374 + | ConversationDataset.SUPPORTED_DATASET_PATHS):
375 + assert args.backend == "vllm-chat", f"{args.dataset_path} needs to use vllm-chat as the backend." #noqa: E501
376 + elif args.dataset_path in (InstructCoderDataset.SUPPORTED_DATASET_PATHS
377 + | AIMODataset.SUPPORTED_DATASET_PATHS):
378 + assert args.backend == "vllm", f"{args.dataset_path} needs to use vllm as the backend." #noqa: E501
379 + else:
380 + raise ValueError(
381 + f"{args.dataset_path} is not supported by hf dataset.")
382 +
383 + # --random-range-ratio: only used when dataset_name is 'random'
384 + if args.dataset_name != 'random' and args.random_range_ratio is not None:
385 + warnings.warn("--random-range-ratio will be ignored since \
386 + --dataset-name is not 'random'.",
387 + stacklevel=2)
388 +
389 + # --prefix-len: only used when dataset_name is 'random', 'sonnet', or not
390 + # set.
391 + if args.dataset_name not in {"random", "sonnet", None
392 + } and args.prefix_len is not None:
393 + warnings.warn("--prefix-len will be ignored since --dataset-name\
394 + is not 'random', 'sonnet', or not set.",
395 + stacklevel=2)
396 +
397 + # === LoRA Settings ===
398 + if getattr(args, "enable_lora", False) and args.backend != "vllm":
399 + raise ValueError(
400 + "LoRA benchmarking is only supported for vLLM backend")
401 + if getattr(args, "enable_lora", False) and args.lora_path is None:
402 + raise ValueError("LoRA path must be provided when enable_lora is True")
403 +
404 + # === Backend-specific Validations ===
405 + if args.backend == "hf" and args.hf_max_batch_size is None:
406 + raise ValueError("HF max batch size is required for HF backend")
407 + if args.backend != "hf" and args.hf_max_batch_size is not None:
408 + raise ValueError("HF max batch size is only for HF backend.")
409 +
410 + if args.backend in {"hf", "mii"} and getattr(args, "quantization",
411 + None) is not None:
412 + raise ValueError("Quantization is only for vLLM backend.")
413 +
414 + if args.backend == "mii" and args.dtype != "auto":
415 + raise ValueError("dtype must be auto for MII backend.")
416 + if args.backend == "mii" and args.n != 1:
417 + raise ValueError("n must be 1 for MII backend.")
418 + if args.backend == "mii" and args.tokenizer != args.model:
419 + raise ValueError(
420 + "Tokenizer must be the same as the model for MII backend.")
421 +
422 +
423 +def add_cli_args(parser: argparse.ArgumentParser):
424 + parser.add_argument("--backend",
425 + type=str,
426 + choices=["vllm", "hf", "mii", "vllm-chat"],
427 + default="vllm")
428 + parser.add_argument(
429 + "--dataset-name",
430 + type=str,
431 + choices=["sharegpt", "random", "sonnet", "burstgpt", "hf"],
432 + help="Name of the dataset to benchmark on.",
433 + default="sharegpt")
434 + parser.add_argument(
435 + "--dataset",
436 + type=str,
437 + default=None,
438 + help="Path to the ShareGPT dataset, will be deprecated in\
439 + the next release. The dataset is expected to "
440 + "be a json in form of list[dict[..., conversations: "
441 + "list[dict[..., value: <prompt_or_response>]]]]")
442 + parser.add_argument("--dataset-path",
443 + type=str,
444 + default=None,
445 + help="Path to the dataset")
446 + parser.add_argument("--input-len",
447 + type=int,
448 + default=None,
449 + help="Input prompt length for each request")
450 + parser.add_argument("--output-len",
451 + type=int,
452 + default=None,
453 + help="Output length for each request. Overrides the "
454 + "output length from the dataset.")
455 + parser.add_argument("--n",
456 + type=int,
457 + default=1,
458 + help="Number of generated sequences per prompt.")
459 + parser.add_argument("--num-prompts",
460 + type=int,
461 + default=1000,
462 + help="Number of prompts to process.")
463 + parser.add_argument("--hf-max-batch-size",
464 + type=int,
465 + default=None,
466 + help="Maximum batch size for HF backend.")
467 + parser.add_argument(
468 + '--output-json',
469 + type=str,
470 + default=None,
471 + help='Path to save the throughput results in JSON format.')
472 + parser.add_argument("--async-engine",
473 + action='store_true',
474 + default=False,
475 + help="Use vLLM async engine rather than LLM class.")
476 + parser.add_argument("--disable-frontend-multiprocessing",
477 + action='store_true',
478 + default=False,
479 + help="Disable decoupled async engine frontend.")
480 + parser.add_argument(
481 + "--disable-detokenize",
482 + action="store_true",
483 + help=("Do not detokenize the response (i.e. do not include "
484 + "detokenization time in the measurement)"))
485 + # LoRA
486 + parser.add_argument(
487 + "--lora-path",
488 + type=str,
489 + default=None,
490 + help="Path to the lora adapters to use. This can be an absolute path, "
491 + "a relative path, or a Hugging Face model identifier.")
492 + parser.add_argument(
493 + "--prefix-len",
494 + type=int,
495 + default=0,
496 + help="Number of fixed prefix tokens before the random "
497 + "context in a request (default: 0).",
498 + )
499 + # random dataset
500 + parser.add_argument(
501 + "--random-range-ratio",
502 + type=float,
503 + default=0.0,
504 + help="Range ratio for sampling input/output length, "
505 + "used only for RandomDataset. Must be in the range [0, 1) to define "
506 + "a symmetric sampling range "
507 + "[length * (1 - range_ratio), length * (1 + range_ratio)].",
508 + )
509 +
510 + # hf dtaset
511 + parser.add_argument("--hf-subset",
512 + type=str,
513 + default=None,
514 + help="Subset of the HF dataset.")
515 + parser.add_argument("--hf-split",
516 + type=str,
517 + default=None,
518 + help="Split of the HF dataset.")
519 +
520 + parser = AsyncEngineArgs.add_cli_args(parser)
521 +
522 +
523 +def main(args: argparse.Namespace):
524 + if args.tokenizer is None:
525 + args.tokenizer = args.model
526 + validate_args(args)
527 + if args.seed is None:
528 + args.seed = 0
529 + print(args)
530 + random.seed(args.seed)
531 + # Sample the requests.
532 + tokenizer = AutoTokenizer.from_pretrained(
533 + args.tokenizer, trust_remote_code=args.trust_remote_code)
534 + requests = get_requests(args, tokenizer)
535 + is_multi_modal = any(request.multi_modal_data is not None
536 + for request in requests)
537 + request_outputs: Optional[list[RequestOutput]] = None
538 + if args.backend == "vllm":
539 + if args.async_engine:
540 + elapsed_time = uvloop.run(
541 + run_vllm_async(
542 + requests,
543 + args.n,
544 + AsyncEngineArgs.from_cli_args(args),
545 + args.disable_frontend_multiprocessing,
546 + args.disable_detokenize,
547 + ))
548 + else:
549 + elapsed_time, request_outputs = run_vllm(
550 + requests, args.n, EngineArgs.from_cli_args(args),
551 + args.disable_detokenize)
552 + elif args.backend == "hf":
553 + assert args.tensor_parallel_size == 1
554 + elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
555 + args.hf_max_batch_size, args.trust_remote_code,
556 + args.disable_detokenize)
557 + elif args.backend == "vllm-chat":
558 + elapsed_time, request_outputs = run_vllm_chat(
559 + requests, args.n, EngineArgs.from_cli_args(args),
560 + args.disable_detokenize)
561 + else:
562 + raise ValueError(f"Unknown backend: {args.backend}")
563 +
564 + if request_outputs:
565 + # Note: with the vllm and vllm-chat backends,
566 + # we have request_outputs, which we use to count tokens.
567 + total_prompt_tokens = 0
568 + total_output_tokens = 0
569 + for ro in request_outputs:
570 + if not isinstance(ro, RequestOutput):
571 + continue
572 + total_prompt_tokens += len(
573 + ro.prompt_token_ids) if ro.prompt_token_ids else 0
574 + total_output_tokens += sum(
575 + len(o.token_ids) for o in ro.outputs if o)
576 + total_num_tokens = total_prompt_tokens + total_output_tokens
577 + else:
578 + total_num_tokens = sum(r.prompt_len + r.expected_output_len
579 + for r in requests)
580 + total_output_tokens = sum(r.expected_output_len for r in requests)
581 + total_prompt_tokens = total_num_tokens - total_output_tokens
582 +
583 + if is_multi_modal and args.backend != "vllm-chat":
584 + print("\033[91mWARNING\033[0m: Multi-modal request with "
585 + f"{args.backend} backend detected. The "
586 + "following metrics are not accurate because image tokens are not"
587 + " counted. See vllm-project/vllm/issues/9778 for details.")
588 + # TODO(vllm-project/vllm/issues/9778): Count multi-modal token length.
589 + # vllm-chat backend counts the image tokens now
590 +
591 + print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
592 + f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
593 + f"{total_output_tokens / elapsed_time:.2f} output tokens/s")
594 + print(f"Total num prompt tokens: {total_prompt_tokens}")
595 + print(f"Total num output tokens: {total_output_tokens}")
596 +
597 + # Output JSON results if specified
598 + if args.output_json:
599 + results = {
600 + "elapsed_time": elapsed_time,
601 + "num_requests": len(requests),
602 + "total_num_tokens": total_num_tokens,
603 + "requests_per_second": len(requests) / elapsed_time,
604 + "tokens_per_second": total_num_tokens / elapsed_time,
605 + }
606 + with open(args.output_json, "w") as f:
607 + json.dump(results, f, indent=4)
608 + save_to_pytorch_benchmark_format(args, results)
vllm/entrypoints/cli/benchmark/latency.py new
+29
@@ -0,0 +1,29 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import argparse
3 +
4 +from vllm.benchmarks.latency import add_cli_args, main
5 +from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
6 +from vllm.entrypoints.cli.types import CLISubcommand
7 +
8 +
9 +class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
10 + """ The `latency` subcommand for vllm bench. """
11 +
12 + def __init__(self):
13 + self.name = "latency"
14 + super().__init__()
15 +
16 + @property
17 + def help(self) -> str:
18 + return "Benchmark the latency of a single batch of requests."
19 +
20 + def add_cli_args(self, parser: argparse.ArgumentParser) -> None:
21 + add_cli_args(parser)
22 +
23 + @staticmethod
24 + def cmd(args: argparse.Namespace) -> None:
25 + main(args)
26 +
27 +
28 +def cmd_init() -> list[CLISubcommand]:
29 + return [BenchmarkLatencySubcommand()]
vllm/entrypoints/cli/benchmark/main.py
+4 -2
@@ -1,14 +1,16 @@
1 # SPDX-License-Identifier: Apache-2.0
2 import argparse
3
4 +import vllm.entrypoints.cli.benchmark.latency
5 import vllm.entrypoints.cli.benchmark.serve
6 +import vllm.entrypoints.cli.benchmark.throughput
7 from vllm.entrypoints.cli.types import CLISubcommand
8 from vllm.utils import FlexibleArgumentParser
9
8 -# TODO: Add the rest of the benchmark subcommands here,
9 -# e.g., throughput, latency, etc.
10 BENCHMARK_CMD_MODULES = [
11 + vllm.entrypoints.cli.benchmark.latency,
12 vllm.entrypoints.cli.benchmark.serve,
13 + vllm.entrypoints.cli.benchmark.throughput,
14 ]
15
16
vllm/entrypoints/cli/benchmark/throughput.py new
+29
@@ -0,0 +1,29 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import argparse
3 +
4 +from vllm.benchmarks.throughput import add_cli_args, main
5 +from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
6 +from vllm.entrypoints.cli.types import CLISubcommand
7 +
8 +
9 +class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
10 + """ The `throughput` subcommand for vllm bench. """
11 +
12 + def __init__(self):
13 + self.name = "throughput"
14 + super().__init__()
15 +
16 + @property
17 + def help(self) -> str:
18 + return "Benchmark offline inference throughput."
19 +
20 + def add_cli_args(self, parser: argparse.ArgumentParser) -> None:
21 + add_cli_args(parser)
22 +
23 + @staticmethod
24 + def cmd(args: argparse.Namespace) -> None:
25 + main(args)
26 +
27 +
28 +def cmd_init() -> list[CLISubcommand]:
29 + return [BenchmarkThroughputSubcommand()]