@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 36e0c8f7da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 19:45 UTC

[Feature] Add `vllm bench` CLI (#13993)

Signed-off-by: Randy Chen <acad.randyjhc@gmail.com> Signed-off-by: Cody Yu <hao.yu.cody@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Randy Chen committed Mar 11, 2025 at 17:31 UTC 36e0c8f7da96b69abf7e8b19527c0ee953805395
8 files changed +1274
vllm/benchmarks/endpoint_request_func.py new
+160
@@ -0,0 +1,160 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""The request function for API endpoints."""
3 +
4 +import json
5 +import os
6 +import sys
7 +import time
8 +import traceback
9 +from dataclasses import dataclass, field
10 +from typing import Optional
11 +
12 +import aiohttp
13 +from tqdm.asyncio import tqdm
14 +
15 +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
16 +
17 +
18 +@dataclass
19 +class RequestFuncInput:
20 + """The input for the request function."""
21 + prompt: str
22 + api_url: str
23 + prompt_len: int
24 + output_len: int
25 + model: str
26 + model_name: Optional[str] = None
27 + best_of: int = 1
28 + logprobs: Optional[int] = None
29 + extra_body: Optional[dict] = None
30 + multi_modal_content: Optional[dict] = None
31 + ignore_eos: bool = False
32 +
33 +
34 +@dataclass
35 +class RequestFuncOutput:
36 + """The output of the request function including metrics."""
37 + generated_text: str = ""
38 + success: bool = False
39 + latency: float = 0.0
40 + output_tokens: int = 0
41 + ttft: float = 0.0 # Time to first token
42 + itl: list[float] = field(
43 + default_factory=list) # list of inter-token latencies
44 + tpot: float = 0.0 # avg next-token latencies
45 + prompt_len: int = 0
46 + error: str = ""
47 +
48 +
49 +async def async_request_openai_completions(
50 + request_func_input: RequestFuncInput,
51 + pbar: Optional[tqdm] = None,
52 +) -> RequestFuncOutput:
53 + """The async request function for the OpenAI Completions API.
54 +
55 + Args:
56 + request_func_input: The input for the request function.
57 + pbar: The progress bar to display the progress.
58 +
59 + Returns:
60 + The output of the request function.
61 + """
62 + api_url = request_func_input.api_url
63 + assert api_url.endswith(
64 + ("completions", "profile")
65 + ), "OpenAI Completions API URL must end with 'completions' or 'profile'."
66 +
67 + async with aiohttp.ClientSession(trust_env=True,
68 + timeout=AIOHTTP_TIMEOUT) as session:
69 + payload = {
70 + "model": request_func_input.model_name \
71 + if request_func_input.model_name else request_func_input.model,
72 + "prompt": request_func_input.prompt,
73 + "temperature": 0.0,
74 + "best_of": request_func_input.best_of,
75 + "max_tokens": request_func_input.output_len,
76 + "logprobs": request_func_input.logprobs,
77 + "stream": True,
78 + "stream_options": {
79 + "include_usage": True,
80 + },
81 + }
82 + if request_func_input.ignore_eos:
83 + payload["ignore_eos"] = request_func_input.ignore_eos
84 + if request_func_input.extra_body:
85 + payload.update(request_func_input.extra_body)
86 + headers = {
87 + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
88 + }
89 +
90 + output = RequestFuncOutput()
91 + output.prompt_len = request_func_input.prompt_len
92 +
93 + generated_text = ""
94 + st = time.perf_counter()
95 + most_recent_timestamp = st
96 + try:
97 + async with session.post(url=api_url, json=payload,
98 + headers=headers) as response:
99 + if response.status == 200:
100 + first_chunk_received = False
101 + async for chunk_bytes in response.content:
102 + chunk_bytes = chunk_bytes.strip()
103 + if not chunk_bytes:
104 + continue
105 +
106 + chunk = chunk_bytes.decode("utf-8").removeprefix(
107 + "data: ")
108 + if chunk != "[DONE]":
109 + data = json.loads(chunk)
110 +
111 + # NOTE: Some completion API might have a last
112 + # usage summary response without a token so we
113 + # want to check a token was generated
114 + if choices := data.get("choices"):
115 + # Note that text could be empty here
116 + # e.g. for special tokens
117 + text = choices[0].get("text")
118 + timestamp = time.perf_counter()
119 + # First token
120 + if not first_chunk_received:
121 + first_chunk_received = True
122 + ttft = time.perf_counter() - st
123 + output.ttft = ttft
124 +
125 + # Decoding phase
126 + else:
127 + output.itl.append(timestamp -
128 + most_recent_timestamp)
129 +
130 + most_recent_timestamp = timestamp
131 + generated_text += text or ""
132 + elif usage := data.get("usage"):
133 + output.output_tokens = usage.get(
134 + "completion_tokens")
135 + if first_chunk_received:
136 + output.success = True
137 + else:
138 + output.success = False
139 + output.error = (
140 + "Never received a valid chunk to calculate TTFT."
141 + "This response will be marked as failed!")
142 + output.generated_text = generated_text
143 + output.latency = most_recent_timestamp - st
144 + else:
145 + output.error = response.reason or ""
146 + output.success = False
147 + except Exception:
148 + output.success = False
149 + exc_info = sys.exc_info()
150 + output.error = "".join(traceback.format_exception(*exc_info))
151 +
152 + if pbar:
153 + pbar.update(1)
154 + return output
155 +
156 +
157 +# TODO: Add more request functions for different API protocols.
158 +ASYNC_REQUEST_FUNCS = {
159 + "openai-comp": async_request_openai_completions,
160 +}
vllm/benchmarks/serve.py new
+927
@@ -0,0 +1,927 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +r"""Benchmark online serving throughput.
3 +
4 +On the server side, run one of the following commands
5 +to launch the vLLM OpenAI API server:
6 + vllm serve <your_model> <engine arguments>
7 +
8 +On the client side, run:
9 + vllm bench serve \
10 + --endpoint-type <endpoint_type. Default 'openi-comp'> \
11 + --label <benchmark result label. Default using endpoint_type> \
12 + --model <your_model> \
13 + --dataset-name <dataset_name. Default 'random'> \
14 + --request-rate <request_rate. Default inf> \
15 + --num-prompts <num_prompts. Default 1000>
16 +"""
17 +import argparse
18 +import asyncio
19 +import gc
20 +import json
21 +import os
22 +import random
23 +import time
24 +import warnings
25 +from collections.abc import AsyncGenerator
26 +from dataclasses import dataclass
27 +from datetime import datetime
28 +from typing import Any, Optional
29 +
30 +import numpy as np
31 +from tqdm.asyncio import tqdm
32 +from transformers import PreTrainedTokenizerBase
33 +
34 +from vllm.benchmarks.endpoint_request_func import (ASYNC_REQUEST_FUNCS,
35 + RequestFuncInput,
36 + RequestFuncOutput)
37 +from vllm.benchmarks.utils import (convert_to_pytorch_benchmark_format,
38 + write_to_json)
39 +from vllm.transformers_utils.tokenizer import get_tokenizer
40 +
41 +MILLISECONDS_TO_SECONDS_CONVERSION = 1000
42 +
43 +
44 +@dataclass
45 +class BenchmarkMetrics:
46 + completed: int
47 + total_input: int
48 + total_output: int
49 + request_throughput: float
50 + request_goodput: float
51 + output_throughput: float
52 + total_token_throughput: float
53 + mean_ttft_ms: float
54 + median_ttft_ms: float
55 + std_ttft_ms: float
56 + percentiles_ttft_ms: list[tuple[float, float]]
57 + mean_tpot_ms: float
58 + median_tpot_ms: float
59 + std_tpot_ms: float
60 + percentiles_tpot_ms: list[tuple[float, float]]
61 + mean_itl_ms: float
62 + median_itl_ms: float
63 + std_itl_ms: float
64 + percentiles_itl_ms: list[tuple[float, float]]
65 + # E2EL stands for end-to-end latency per request.
66 + # It is the time taken on the client side from sending
67 + # a request to receiving a complete response.
68 + mean_e2el_ms: float
69 + median_e2el_ms: float
70 + std_e2el_ms: float
71 + percentiles_e2el_ms: list[tuple[float, float]]
72 +
73 +
74 +def sample_random_requests(
75 + prefix_len: int,
76 + input_len: int,
77 + output_len: int,
78 + num_prompts: int,
79 + range_ratio: float,
80 + tokenizer: PreTrainedTokenizerBase,
81 +) -> list[tuple[str, int, int]]:
82 + prefix_token_ids = np.random.randint(0,
83 + tokenizer.vocab_size,
84 + size=prefix_len).tolist()
85 +
86 + input_lens = np.random.randint(
87 + int(input_len * range_ratio),
88 + input_len + 1,
89 + size=num_prompts,
90 + )
91 + output_lens = np.random.randint(
92 + int(output_len * range_ratio),
93 + output_len + 1,
94 + size=num_prompts,
95 + )
96 + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
97 + input_requests = []
98 + for i in range(num_prompts):
99 + prompt = tokenizer.decode(prefix_token_ids +
100 + [(offsets[i] + i + j) % tokenizer.vocab_size
101 + for j in range(input_lens[i])])
102 +
103 + input_requests.append((prompt, int(prefix_len + input_lens[i]),
104 + int(output_lens[i]), None))
105 +
106 + return input_requests
107 +
108 +
109 +async def get_request(
110 + input_requests: list[tuple[str, int, int]],
111 + request_rate: float,
112 + burstiness: float = 1.0,
113 +) -> AsyncGenerator[tuple[str, int, int], None]:
114 + """
115 + Asynchronously generates requests at a specified rate
116 + with OPTIONAL burstiness.
117 +
118 + Args:
119 + input_requests:
120 + A list of input requests, each represented as a tuple.
121 + request_rate:
122 + The rate at which requests are generated (requests/s).
123 + burstiness (optional):
124 + The burstiness factor of the request generation.
125 + Only takes effect when request_rate is not inf.
126 + Default value is 1, which follows a Poisson process.
127 + Otherwise, the request intervals follow a gamma distribution.
128 + A lower burstiness value (0 < burstiness < 1) results
129 + in more bursty requests, while a higher burstiness value
130 + (burstiness > 1) results in a more uniform arrival of requests.
131 + """
132 + input_requests = iter(input_requests)
133 +
134 + # Calculate scale parameter theta to maintain the desired request_rate.
135 + assert burstiness > 0, (
136 + f"A positive burstiness factor is expected, but given {burstiness}.")
137 + theta = 1.0 / (request_rate * burstiness)
138 +
139 + for request in input_requests:
140 + yield request
141 +
142 + if request_rate == float("inf"):
143 + # If the request rate is infinity, then we don't need to wait.
144 + continue
145 +
146 + # Sample the request interval from the gamma distribution.
147 + # If burstiness is 1, it follows exponential distribution.
148 + interval = np.random.gamma(shape=burstiness, scale=theta)
149 + # The next request will be sent after the interval.
150 + await asyncio.sleep(interval)
151 +
152 +
153 +def calculate_metrics(
154 + input_requests: list[tuple[str, int, int]],
155 + outputs: list[RequestFuncOutput],
156 + dur_s: float,
157 + tokenizer: PreTrainedTokenizerBase,
158 + selected_percentiles: list[float],
159 + goodput_config_dict: dict[str, float],
160 +) -> tuple[BenchmarkMetrics, list[int]]:
161 + """Calculate the metrics for the benchmark.
162 +
163 + Args:
164 + input_requests: The input requests.
165 + outputs: The outputs of the requests.
166 + dur_s: The duration of the benchmark.
167 + tokenizer: The tokenizer to use.
168 + selected_percentiles: The percentiles to select.
169 + goodput_config_dict: The goodput configuration.
170 +
171 + Returns:
172 + A tuple of the benchmark metrics and the actual output lengths.
173 + """
174 + actual_output_lens: list[int] = []
175 + total_input = 0
176 + completed = 0
177 + good_completed = 0
178 + itls: list[float] = []
179 + tpots: list[float] = []
180 + all_tpots: list[float] = []
181 + ttfts: list[float] = []
182 + e2els: list[float] = []
183 + for i in range(len(outputs)):
184 + if outputs[i].success:
185 + output_len = outputs[i].output_tokens
186 +
187 + if output_len is None:
188 + # We use the tokenizer to count the number of output tokens
189 + # for some serving backends instead of looking at
190 + # len(outputs[i].itl) since multiple output tokens may be
191 + # bundled together
192 + # Note : this may inflate the output token count slightly
193 + output_len = len(
194 + tokenizer(outputs[i].generated_text,
195 + add_special_tokens=False).input_ids)
196 + actual_output_lens.append(output_len)
197 + total_input += input_requests[i][1]
198 + tpot = 0
199 + if output_len > 1:
200 + latency_minus_ttft = outputs[i].latency - outputs[i].ttft
201 + tpot = latency_minus_ttft / (output_len - 1)
202 + tpots.append(tpot)
203 + # Note: if output_len <= 1, we regard tpot as 0 for goodput
204 + all_tpots.append(tpot)
205 + itls += outputs[i].itl
206 + ttfts.append(outputs[i].ttft)
207 + e2els.append(outputs[i].latency)
208 + completed += 1
209 + else:
210 + actual_output_lens.append(0)
211 +
212 + if goodput_config_dict:
213 + valid_metrics = []
214 + slo_values = []
215 +
216 + if "ttft" in goodput_config_dict:
217 + valid_metrics.append(ttfts)
218 + slo_values.append(goodput_config_dict["ttft"] /
219 + MILLISECONDS_TO_SECONDS_CONVERSION)
220 + if "tpot" in goodput_config_dict:
221 + valid_metrics.append(all_tpots)
222 + slo_values.append(goodput_config_dict["tpot"] /
223 + MILLISECONDS_TO_SECONDS_CONVERSION)
224 + if "e2el" in goodput_config_dict:
225 + valid_metrics.append(e2els)
226 + slo_values.append(goodput_config_dict["e2el"] /
227 + MILLISECONDS_TO_SECONDS_CONVERSION)
228 +
229 + for req_metric in zip(*valid_metrics):
230 + is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])
231 + if is_good_req:
232 + good_completed += 1
233 +
234 + if completed == 0:
235 + warnings.warn(
236 + "All requests failed. This is likely due to a misconfiguration "
237 + "on the benchmark arguments.",
238 + stacklevel=2)
239 + metrics = BenchmarkMetrics(
240 + completed=completed,
241 + total_input=total_input,
242 + total_output=sum(actual_output_lens),
243 + request_throughput=completed / dur_s,
244 + request_goodput=good_completed / dur_s,
245 + output_throughput=sum(actual_output_lens) / dur_s,
246 + total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
247 + mean_ttft_ms=np.mean(ttfts or 0) *
248 + 1000, # ttfts is empty if streaming is not supported by the endpoint
249 + std_ttft_ms=np.std(ttfts or 0) * 1000,
250 + median_ttft_ms=np.median(ttfts or 0) * 1000,
251 + percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000)
252 + for p in selected_percentiles],
253 + mean_tpot_ms=np.mean(tpots or 0) * 1000,
254 + std_tpot_ms=np.std(tpots or 0) * 1000,
255 + median_tpot_ms=np.median(tpots or 0) * 1000,
256 + percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000)
257 + for p in selected_percentiles],
258 + mean_itl_ms=np.mean(itls or 0) * 1000,
259 + std_itl_ms=np.std(itls or 0) * 1000,
260 + median_itl_ms=np.median(itls or 0) * 1000,
261 + percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
262 + for p in selected_percentiles],
263 + mean_e2el_ms=np.mean(e2els or 0) * 1000,
264 + std_e2el_ms=np.std(e2els or 0) * 1000,
265 + median_e2el_ms=np.median(e2els or 0) * 1000,
266 + percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
267 + for p in selected_percentiles],
268 + )
269 +
270 + return metrics, actual_output_lens
271 +
272 +
273 +async def benchmark(
274 + endpoint_type: str,
275 + api_url: str,
276 + base_url: str,
277 + model_id: str,
278 + model_name: str,
279 + tokenizer: PreTrainedTokenizerBase,
280 + input_requests: list[tuple[str, int, int]],
281 + logprobs: Optional[int],
282 + best_of: int,
283 + request_rate: float,
284 + burstiness: float,
285 + disable_tqdm: bool,
286 + profile: bool,
287 + selected_percentile_metrics: list[str],
288 + selected_percentiles: list[str],
289 + ignore_eos: bool,
290 + goodput_config_dict: dict[str, float],
291 + max_concurrency: Optional[int],
292 + lora_modules: Optional[list[str]],
293 +):
294 + if endpoint_type in ASYNC_REQUEST_FUNCS:
295 + request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
296 + else:
297 + raise ValueError(f"Unknown endpoint_type: {endpoint_type}")
298 +
299 + print("Starting initial single prompt test run...")
300 + test_prompt, test_prompt_len, test_output_len, test_mm_content = (
301 + input_requests[0])
302 + if endpoint_type != "openai-chat" and test_mm_content is not None:
303 + # multi-modal benchmark is only available on OpenAI Chat endpoint.
304 + raise ValueError("Multi-modal content is only supported on "
305 + "'openai-chat' endpoint_type.")
306 + test_input = RequestFuncInput(
307 + model=model_id,
308 + model_name=model_name,
309 + prompt=test_prompt,
310 + api_url=api_url,
311 + prompt_len=test_prompt_len,
312 + output_len=test_output_len,
313 + logprobs=logprobs,
314 + best_of=best_of,
315 + multi_modal_content=test_mm_content,
316 + ignore_eos=ignore_eos,
317 + )
318 +
319 + test_output = await request_func(request_func_input=test_input)
320 + if not test_output.success:
321 + raise ValueError(
322 + "Initial test run failed - Please make sure benchmark arguments "
323 + f"are correctly specified. Error: {test_output.error}")
324 + else:
325 + print("Initial test run completed. Starting main benchmark run...")
326 +
327 + if lora_modules:
328 + # For each input request, choose a LoRA module at random.
329 + lora_modules = iter(
330 + [random.choice(lora_modules) for _ in range(len(input_requests))])
331 +
332 + if profile:
333 + print("Starting profiler...")
334 + profile_input = RequestFuncInput(model=model_id,
335 + model_name=model_name,
336 + prompt=test_prompt,
337 + api_url=base_url + "/start_profile",
338 + prompt_len=test_prompt_len,
339 + output_len=test_output_len,
340 + logprobs=logprobs,
341 + best_of=best_of,
342 + multi_modal_content=test_mm_content,
343 + ignore_eos=ignore_eos)
344 + profile_output = await request_func(request_func_input=profile_input)
345 + if profile_output.success:
346 + print("Profiler started")
347 +
348 + if burstiness == 1.0:
349 + distribution = "Poisson process"
350 + else:
351 + distribution = "Gamma distribution"
352 +
353 + print(f"Traffic request rate: {request_rate}")
354 + print(f"Burstiness factor: {burstiness} ({distribution})")
355 + print(f"Maximum request concurrency: {max_concurrency}")
356 +
357 + pbar = None if disable_tqdm else tqdm(total=len(input_requests))
358 +
359 + # This can be used once the minimum Python version is 3.10 or higher,
360 + # and it will simplify the code in limited_request_func.
361 + # semaphore = (asyncio.Semaphore(max_concurrency)
362 + # if max_concurrency else contextlib.nullcontext())
363 + semaphore = (asyncio.Semaphore(max_concurrency)
364 + if max_concurrency else None)
365 +
366 + async def limited_request_func(request_func_input, pbar):
367 + if semaphore is None:
368 + return await request_func(request_func_input=request_func_input,
369 + pbar=pbar)
370 + async with semaphore:
371 + return await request_func(request_func_input=request_func_input,
372 + pbar=pbar)
373 +
374 + benchmark_start_time = time.perf_counter()
375 + tasks: list[asyncio.Task] = []
376 + async for request in get_request(input_requests, request_rate, burstiness):
377 + prompt, prompt_len, output_len, mm_content = request
378 + req_model_id, req_model_name = model_id, model_name
379 + if lora_modules:
380 + req_lora_module = next(lora_modules)
381 + req_model_id, req_model_name = req_lora_module, req_lora_module
382 +
383 + request_func_input = RequestFuncInput(model=req_model_id,
384 + model_name=req_model_name,
385 + prompt=prompt,
386 + api_url=api_url,
387 + prompt_len=prompt_len,
388 + output_len=output_len,
389 + logprobs=logprobs,
390 + best_of=best_of,
391 + multi_modal_content=mm_content,
392 + ignore_eos=ignore_eos)
393 + tasks.append(
394 + asyncio.create_task(
395 + limited_request_func(request_func_input=request_func_input,
396 + pbar=pbar)))
397 + outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)
398 +
399 + if profile:
400 + print("Stopping profiler...")
401 + profile_input = RequestFuncInput(
402 + model=model_id,
403 + prompt=test_prompt,
404 + api_url=base_url + "/stop_profile",
405 + prompt_len=test_prompt_len,
406 + output_len=test_output_len,
407 + logprobs=logprobs,
408 + best_of=best_of,
409 + )
410 + profile_output = await request_func(request_func_input=profile_input)
411 + if profile_output.success:
412 + print("Profiler stopped")
413 +
414 + if pbar is not None:
415 + pbar.close()
416 +
417 + benchmark_duration = time.perf_counter() - benchmark_start_time
418 +
419 + metrics, actual_output_lens = calculate_metrics(
420 + input_requests=input_requests,
421 + outputs=outputs,
422 + dur_s=benchmark_duration,
423 + tokenizer=tokenizer,
424 + selected_percentiles=selected_percentiles,
425 + goodput_config_dict=goodput_config_dict,
426 + )
427 +
428 + print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
429 + print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
430 + print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
431 + benchmark_duration))
432 + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
433 + print("{:<40} {:<10}".format("Total generated tokens:",
434 + metrics.total_output))
435 + print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
436 + metrics.request_throughput))
437 + if goodput_config_dict:
438 + print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
439 + metrics.request_goodput))
440 + print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
441 + metrics.output_throughput))
442 + print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
443 + metrics.total_token_throughput))
444 +
445 + result = {
446 + "duration": benchmark_duration,
447 + "completed": metrics.completed,
448 + "total_input_tokens": metrics.total_input,
449 + "total_output_tokens": metrics.total_output,
450 + "request_throughput": metrics.request_throughput,
451 + "request_goodput:":
452 + metrics.request_goodput if goodput_config_dict else None,
453 + "output_throughput": metrics.output_throughput,
454 + "total_token_throughput": metrics.total_token_throughput,
455 + "input_lens": [output.prompt_len for output in outputs],
456 + "output_lens": actual_output_lens,
457 + "ttfts": [output.ttft for output in outputs],
458 + "itls": [output.itl for output in outputs],
459 + "generated_texts": [output.generated_text for output in outputs],
460 + "errors": [output.error for output in outputs],
461 + }
462 +
463 + def process_one_metric(
464 + # E.g., "ttft"
465 + metric_attribute_name: str,
466 + # E.g., "TTFT"
467 + metric_name: str,
468 + # E.g., "Time to First Token"
469 + metric_header: str,
470 + ):
471 + # This function prints and adds statistics of the specified
472 + # metric.
473 + if metric_attribute_name not in selected_percentile_metrics:
474 + return
475 + print("{s:{c}^{n}}".format(s=metric_header, n=50, c='-'))
476 + print("{:<40} {:<10.2f}".format(
477 + f"Mean {metric_name} (ms):",
478 + getattr(metrics, f"mean_{metric_attribute_name}_ms")))
479 + print("{:<40} {:<10.2f}".format(
480 + f"Median {metric_name} (ms):",
481 + getattr(metrics, f"median_{metric_attribute_name}_ms")))
482 + result[f"mean_{metric_attribute_name}_ms"] = getattr(
483 + metrics, f"mean_{metric_attribute_name}_ms")
484 + result[f"median_{metric_attribute_name}_ms"] = getattr(
485 + metrics, f"median_{metric_attribute_name}_ms")
486 + result[f"std_{metric_attribute_name}_ms"] = getattr(
487 + metrics, f"std_{metric_attribute_name}_ms")
488 + for p, value in getattr(metrics,
489 + f"percentiles_{metric_attribute_name}_ms"):
490 + p_word = str(int(p)) if int(p) == p else str(p)
491 + print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):",
492 + value))
493 + result[f"p{p_word}_{metric_attribute_name}_ms"] = value
494 +
495 + process_one_metric("ttft", "TTFT", "Time to First Token")
496 + process_one_metric("tpot", "TPOT",
497 + "Time per Output Token (excl. 1st token)")
498 + process_one_metric("itl", "ITL", "Inter-token Latency")
499 + process_one_metric("e2el", "E2EL", "End-to-end Latency")
500 +
501 + print("=" * 50)
502 +
503 + return result
504 +
505 +
506 +def check_goodput_args(args):
507 + # Check and parse goodput arguments
508 + goodput_config_dict = {}
509 + VALID_NAMES = ["ttft", "tpot", "e2el"]
510 + if args.goodput:
511 + goodput_config_dict = parse_goodput(args.goodput)
512 + for slo_name, slo_val in goodput_config_dict.items():
513 + if slo_name not in VALID_NAMES:
514 + raise ValueError(
515 + f"Invalid metric name found, {slo_name}: {slo_val}. "
516 + "The service level objective name should be one of "
517 + f"{str(VALID_NAMES)}. ")
518 + if slo_val < 0:
519 + raise ValueError(
520 + f"Invalid value found, {slo_name}: {slo_val}. "
521 + "The service level objective value should be "
522 + "non-negative.")
523 + return goodput_config_dict
524 +
525 +
526 +def parse_goodput(slo_pairs):
527 + goodput_config_dict = {}
528 + try:
529 + for slo_pair in slo_pairs:
530 + slo_name, slo_val = slo_pair.split(":")
531 + goodput_config_dict[slo_name] = float(slo_val)
532 + except ValueError as err:
533 + raise argparse.ArgumentTypeError(
534 + "Invalid format found for service level objectives. "
535 + "Specify service level objectives for goodput as \"KEY:VALUE\" "
536 + "pairs, where the key is a metric name, and the value is a "
537 + "number in milliseconds.") from err
538 + return goodput_config_dict
539 +
540 +
541 +def save_to_pytorch_benchmark_format(args: argparse.Namespace,
542 + results: dict[str, Any],
543 + file_name: str) -> None:
544 + metrics = [
545 + "median_ttft_ms", "mean_ttft_ms", "std_ttft_ms", "p99_ttft_ms",
546 + "mean_tpot_ms", "median_tpot_ms", "std_tpot_ms", "p99_tpot_ms",
547 + "median_itl_ms", "mean_itl_ms", "std_itl_ms", "p99_itl_ms"
548 + ]
549 + # These raw data might be useful, but they are rather big. They can be added
550 + # later if needed
551 + ignored_metrics = ["ttfts", "itls", "generated_texts", "errors"]
552 + pt_records = convert_to_pytorch_benchmark_format(
553 + args=args,
554 + metrics={k: [results[k]]
555 + for k in metrics},
556 + extra_info={
557 + k: results[k]
558 + for k in results if k not in metrics and k not in ignored_metrics
559 + })
560 + if pt_records:
561 + # Don't use json suffix here as we don't want CI to pick it up
562 + pt_file = f"{os.path.splitext(file_name)[0]}.pytorch.json"
563 + write_to_json(pt_file, pt_records)
564 +
565 +
566 +def add_cli_args(parser: argparse.ArgumentParser):
567 + parser.add_argument(
568 + "--endpoint-type",
569 + type=str,
570 + default="openai-comp",
571 + choices=list(ASYNC_REQUEST_FUNCS.keys()),
572 + )
573 + parser.add_argument(
574 + "--label",
575 + type=str,
576 + default=None,
577 + help="The label (prefix) of the benchmark results. If not specified, "
578 + "the endpoint type will be used as the label.",
579 + )
580 + parser.add_argument(
581 + "--base-url",
582 + type=str,
583 + default=None,
584 + help="Server or API base url if not using http host and port.",
585 + )
586 + # Use 127.0.0.1 here instead of localhost to force the use of ipv4
587 + parser.add_argument("--host", type=str, default="127.0.0.1")
588 + parser.add_argument("--port", type=int, default=8000)
589 + parser.add_argument(
590 + "--endpoint",
591 + type=str,
592 + default="/v1/completions",
593 + help="API endpoint.",
594 + )
595 + parser.add_argument(
596 + "--dataset-name",
597 + type=str,
598 + default="random",
599 + choices=["random"],
600 + help="Name of the dataset to benchmark on.",
601 + )
602 + parser.add_argument(
603 + "--max-concurrency",
604 + type=int,
605 + default=None,
606 + help="Maximum number of concurrent requests. This can be used "
607 + "to help simulate an environment where a higher level component "
608 + "is enforcing a maximum number of concurrent requests. While the "
609 + "--request-rate argument controls the rate at which requests are "
610 + "initiated, this argument will control how many are actually allowed "
611 + "to execute at a time. This means that when used in combination, the "
612 + "actual request rate may be lower than specified with --request-rate, "
613 + "if the server is not processing requests fast enough to keep up.")
614 +
615 + parser.add_argument(
616 + "--model",
617 + type=str,
618 + required=True,
619 + help="Name of the model.",
620 + )
621 + parser.add_argument(
622 + "--tokenizer",
623 + type=str,
624 + help=
625 + "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
626 + )
627 + parser.add_argument(
628 + "--best-of",
629 + type=int,
630 + default=1,
631 + help="Generates `best_of` sequences per prompt and "
632 + "returns the best one.",
633 + )
634 + parser.add_argument("--use-beam-search", action="store_true")
635 + parser.add_argument(
636 + "--num-prompts",
637 + type=int,
638 + default=1000,
639 + help="Number of prompts to process.",
640 + )
641 + parser.add_argument(
642 + "--logprobs",
643 + type=int,
644 + default=None,
645 + help=("Number of logprobs-per-token to compute & return as part of "
646 + "the request. If unspecified, then either (1) if beam search "
647 + "is disabled, no logprobs are computed & a single dummy "
648 + "logprob is returned for each token; or (2) if beam search "
649 + "is enabled 1 logprob per token is computed"),
650 + )
651 + parser.add_argument(
652 + "--request-rate",
653 + type=float,
654 + default=float("inf"),
655 + help="Number of requests per second. If this is inf, "
656 + "then all the requests are sent at time 0. "
657 + "Otherwise, we use Poisson process or gamma distribution "
658 + "to synthesize the request arrival times.",
659 + )
660 + parser.add_argument(
661 + "--burstiness",
662 + type=float,
663 + default=1.0,
664 + help="Burstiness factor of the request generation. "
665 + "Only take effect when request_rate is not inf. "
666 + "Default value is 1, which follows Poisson process. "
667 + "Otherwise, the request intervals follow a gamma distribution. "
668 + "A lower burstiness value (0 < burstiness < 1) results in more "
669 + "bursty requests. A higher burstiness value (burstiness > 1) "
670 + "results in a more uniform arrival of requests.",
671 + )
672 + parser.add_argument("--seed", type=int, default=0)
673 + parser.add_argument(
674 + "--trust-remote-code",
675 + action="store_true",
676 + help="Trust remote code from huggingface",
677 + )
678 + parser.add_argument(
679 + "--disable-tqdm",
680 + action="store_true",
681 + help="Specify to disable tqdm progress bar.",
682 + )
683 + parser.add_argument(
684 + "--profile",
685 + action="store_true",
686 + help="Use Torch Profiler. The endpoint must be launched with "
687 + "VLLM_TORCH_PROFILER_DIR to enable profiler.",
688 + )
689 + parser.add_argument(
690 + "--save-result",
691 + action="store_true",
692 + help="Specify to save benchmark results to a json file",
693 + )
694 + parser.add_argument(
695 + "--metadata",
696 + metavar="KEY=VALUE",
697 + nargs="*",
698 + help="Key-value pairs (e.g, --metadata version=0.3.3 tp=1) "
699 + "for metadata of this run to be saved in the result JSON file "
700 + "for record keeping purposes.",
701 + )
702 + parser.add_argument(
703 + "--result-dir",
704 + type=str,
705 + default=None,
706 + help="Specify directory to save benchmark json results."
707 + "If not specified, results are saved in the current directory.",
708 + )
709 + parser.add_argument(
710 + "--result-filename",
711 + type=str,
712 + default=None,
713 + help="Specify the filename to save benchmark json results."
714 + "If not specified, results will be saved in "
715 + "{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" # noqa
716 + " format.",
717 + )
718 + parser.add_argument(
719 + "--ignore-eos",
720 + action="store_true",
721 + help="Set ignore_eos flag when sending the benchmark request."
722 + "Warning: ignore_eos is not supported in deepspeed_mii and tgi.")
723 + parser.add_argument(
724 + "--percentile-metrics",
725 + type=str,
726 + default="ttft,tpot,itl",
727 + help="Comma-seperated list of selected metrics to report percentils. "
728 + "This argument specifies the metrics to report percentiles. "
729 + "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "
730 + "Default value is \"ttft,tpot,itl\".")
731 + parser.add_argument(
732 + "--metric-percentiles",
733 + type=str,
734 + default="99",
735 + help="Comma-seperated list of percentiles for selected metrics. "
736 + "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
737 + "Default value is \"99\". "
738 + "Use \"--percentile-metrics\" to select metrics.",
739 + )
740 + parser.add_argument(
741 + "--goodput",
742 + nargs="+",
743 + required=False,
744 + help="Specify service level objectives for goodput as \"KEY:VALUE\" "
745 + "pairs, where the key is a metric name, and the value is in "
746 + "milliseconds. Multiple \"KEY:VALUE\" pairs can be provided, "
747 + "separated by spaces. Allowed request level metric names are "
748 + "\"ttft\", \"tpot\", \"e2el\". For more context on the definition of "
749 + "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 "
750 + "and the blog: https://hao-ai-lab.github.io/blogs/distserve")
751 +
752 + random_group = parser.add_argument_group("random dataset options")
753 + random_group.add_argument(
754 + "--random-input-len",
755 + type=int,
756 + default=1024,
757 + help=
758 + "Number of input tokens per request, used only for random sampling.",
759 + )
760 + random_group.add_argument(
761 + "--random-output-len",
762 + type=int,
763 + default=128,
764 + help=
765 + "Number of output tokens per request, used only for random sampling.",
766 + )
767 + random_group.add_argument(
768 + "--random-range-ratio",
769 + type=float,
770 + default=1.0,
771 + help="Range of sampled ratio of input/output length, "
772 + "used only for random sampling.",
773 + )
774 + random_group.add_argument(
775 + "--random-prefix-len",
776 + type=int,
777 + default=0,
778 + help="Number of fixed prefix tokens before random "
779 + " context. The length range of context in a random "
780 + " request is [random-prefix-len, "
781 + " random-prefix-len + random-prefix-len * random-range-ratio).")
782 +
783 + parser.add_argument(
784 + '--tokenizer-mode',
785 + type=str,
786 + default="auto",
787 + choices=['auto', 'slow', 'mistral', 'custom'],
788 + help='The tokenizer mode.\n\n* "auto" will use the '
789 + 'fast tokenizer if available.\n* "slow" will '
790 + 'always use the slow tokenizer. \n* '
791 + '"mistral" will always use the `mistral_common` tokenizer. \n*'
792 + '"custom" will use --tokenizer to select the preregistered tokenizer.')
793 +
794 + parser.add_argument("--served-model-name",
795 + type=str,
796 + default=None,
797 + help="The model name used in the API. "
798 + "If not specified, the model name will be the "
799 + "same as the ``--model`` argument. ")
800 +
801 + parser.add_argument("--lora-modules",
802 + nargs='+',
803 + default=None,
804 + help="A subset of LoRA module names passed in when "
805 + "launching the server. For each request, the "
806 + "script chooses a LoRA module at random.")
807 +
808 +
809 +def main(args: argparse.Namespace):
810 + print(args)
811 + random.seed(args.seed)
812 + np.random.seed(args.seed)
813 +
814 + endpoint_type = args.endpoint_type
815 + label = args.label
816 + model_id = args.model
817 + model_name = args.served_model_name
818 + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
819 + tokenizer_mode = args.tokenizer_mode
820 +
821 + if args.base_url is not None:
822 + api_url = f"{args.base_url}{args.endpoint}"
823 + base_url = f"{args.base_url}"
824 + else:
825 + api_url = f"http://{args.host}:{args.port}{args.endpoint}"
826 + base_url = f"http://{args.host}:{args.port}"
827 +
828 + tokenizer = get_tokenizer(tokenizer_id,
829 + tokenizer_mode=tokenizer_mode,
830 + trust_remote_code=args.trust_remote_code)
831 + # TODO: This should be refactored to use the benchmark_dataset.py
832 + # in later PRs.
833 + if args.dataset_name is None:
834 + raise ValueError(
835 + "Please specify '--dataset-name' and the corresponding "
836 + "'--dataset-path' if required.")
837 + elif args.dataset_name == "random":
838 + input_requests = sample_random_requests(
839 + prefix_len=args.random_prefix_len,
840 + input_len=args.random_input_len,
841 + output_len=args.random_output_len,
842 + num_prompts=args.num_prompts,
843 + range_ratio=args.random_range_ratio,
844 + tokenizer=tokenizer,
845 + )
846 +
847 + else:
848 + raise ValueError(f"Unknown dataset: {args.dataset_name}")
849 +
850 + goodput_config_dict = check_goodput_args(args)
851 +
852 + # Avoid GC processing "static" data - reduce pause times.
853 + gc.collect()
854 + gc.freeze()
855 +
856 + benchmark_result = asyncio.run(
857 + benchmark(
858 + endpoint_type=endpoint_type,
859 + api_url=api_url,
860 + base_url=base_url,
861 + model_id=model_id,
862 + model_name=model_name,
863 + tokenizer=tokenizer,
864 + input_requests=input_requests,
865 + logprobs=args.logprobs,
866 + best_of=args.best_of,
867 + request_rate=args.request_rate,
868 + burstiness=args.burstiness,
869 + disable_tqdm=args.disable_tqdm,
870 + profile=args.profile,
871 + selected_percentile_metrics=args.percentile_metrics.split(","),
872 + selected_percentiles=[
873 + float(p) for p in args.metric_percentiles.split(",")
874 + ],
875 + ignore_eos=args.ignore_eos,
876 + goodput_config_dict=goodput_config_dict,
877 + max_concurrency=args.max_concurrency,
878 + lora_modules=args.lora_modules,
879 + ))
880 +
881 + # Save config and results to json
882 + if args.save_result:
883 + result_json: dict[str, Any] = {}
884 +
885 + # Setup
886 + current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
887 + result_json["date"] = current_dt
888 + result_json["endpoint_type"] = endpoint_type
889 + result_json["label"] = label
890 + result_json["model_id"] = model_id
891 + result_json["tokenizer_id"] = tokenizer_id
892 + result_json["best_of"] = args.best_of
893 + result_json["num_prompts"] = args.num_prompts
894 +
895 + # Metadata
896 + if args.metadata:
897 + for item in args.metadata:
898 + if "=" in item:
899 + kvstring = item.split("=")
900 + result_json[kvstring[0].strip()] = kvstring[1].strip()
901 + else:
902 + raise ValueError(
903 + "Invalid metadata format. Please use KEY=VALUE format."
904 + )
905 +
906 + # Traffic
907 + result_json["request_rate"] = (args.request_rate if args.request_rate
908 + < float("inf") else "inf")
909 + result_json["burstiness"] = args.burstiness
910 + result_json["max_concurrency"] = args.max_concurrency
911 +
912 + # Merge with benchmark result
913 + result_json = {**result_json, **benchmark_result}
914 +
915 + # Save to file
916 + base_model_id = model_id.split("/")[-1]
917 + max_concurrency_str = (f"-concurrency{args.max_concurrency}"
918 + if args.max_concurrency is not None else "")
919 + label = label or endpoint_type
920 + file_name = f"{label}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" #noqa
921 + if args.result_filename:
922 + file_name = args.result_filename
923 + if args.result_dir:
924 + file_name = os.path.join(args.result_dir, file_name)
925 + with open(file_name, "w", encoding='utf-8') as outfile:
926 + json.dump(result_json, outfile)
927 + save_to_pytorch_benchmark_format(args, result_json, file_name)
vllm/benchmarks/utils.py new
+69
@@ -0,0 +1,69 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import argparse
4 +import json
5 +import math
6 +import os
7 +from typing import Any
8 +
9 +
10 +def convert_to_pytorch_benchmark_format(args: argparse.Namespace,
11 + metrics: dict[str, list],
12 + extra_info: dict[str, Any]) -> list:
13 + """
14 + Save the benchmark results in the format used by PyTorch OSS benchmark with
15 + on metric per record
16 + https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
17 + """
18 + records = []
19 + if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
20 + return records
21 +
22 + for name, benchmark_values in metrics.items():
23 + record = {
24 + "benchmark": {
25 + "name": "vLLM benchmark",
26 + "extra_info": {
27 + "args": vars(args),
28 + },
29 + },
30 + "model": {
31 + "name": args.model,
32 + },
33 + "metric": {
34 + "name": name,
35 + "benchmark_values": benchmark_values,
36 + "extra_info": extra_info,
37 + },
38 + }
39 +
40 + tp = record["benchmark"]["extra_info"]["args"].get(
41 + "tensor_parallel_size")
42 + # Save tensor_parallel_size parameter if it's part of the metadata
43 + if not tp and "tensor_parallel_size" in extra_info:
44 + record["benchmark"]["extra_info"]["args"][
45 + "tensor_parallel_size"] = extra_info["tensor_parallel_size"]
46 +
47 + records.append(record)
48 +
49 + return records
50 +
51 +
52 +class InfEncoder(json.JSONEncoder):
53 +
54 + def clear_inf(self, o: Any):
55 + if isinstance(o, dict):
56 + return {k: self.clear_inf(v) for k, v in o.items()}
57 + elif isinstance(o, list):
58 + return [self.clear_inf(v) for v in o]
59 + elif isinstance(o, float) and math.isinf(o):
60 + return "inf"
61 + return o
62 +
63 + def iterencode(self, o: Any, *args, **kwargs) -> Any:
64 + return super().iterencode(self.clear_inf(o), *args, **kwargs)
65 +
66 +
67 +def write_to_json(filename: str, records: list) -> None:
68 + with open(filename, "w") as f:
69 + json.dump(records, f, cls=InfEncoder)
vllm/entrypoints/cli/benchmark/__init__.py
vllm/entrypoints/cli/benchmark/base.py new
+37
@@ -0,0 +1,37 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import argparse
3 +
4 +from vllm.entrypoints.cli.types import CLISubcommand
5 +from vllm.utils import FlexibleArgumentParser
6 +
7 +
8 +class BenchmarkSubcommandBase(CLISubcommand):
9 + """ The base class of subcommands for vllm bench. """
10 +
11 + @property
12 + def help(self) -> str:
13 + """The help message of the subcommand."""
14 + raise NotImplementedError
15 +
16 + def add_cli_args(self, parser: argparse.ArgumentParser) -> None:
17 + """Add the CLI arguments to the parser."""
18 + raise NotImplementedError
19 +
20 + @staticmethod
21 + def cmd(args: argparse.Namespace) -> None:
22 + """Run the benchmark.
23 +
24 + Args:
25 + args: The arguments to the command.
26 + """
27 + raise NotImplementedError
28 +
29 + def subparser_init(
30 + self,
31 + subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
32 + parser = subparsers.add_parser(
33 + self.name,
34 + help=self.help,
35 + usage=f"vllm bench {self.name} [options]")
36 + self.add_cli_args(parser)
37 + return parser
vllm/entrypoints/cli/benchmark/main.py new
+50
@@ -0,0 +1,50 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import argparse
3 +
4 +import vllm.entrypoints.cli.benchmark.serve
5 +from vllm.entrypoints.cli.types import CLISubcommand
6 +from vllm.utils import FlexibleArgumentParser
7 +
8 +# TODO: Add the rest of the benchmark subcommands here,
9 +# e.g., throughput, latency, etc.
10 +BENCHMARK_CMD_MODULES = [
11 + vllm.entrypoints.cli.benchmark.serve,
12 +]
13 +
14 +
15 +class BenchmarkSubcommand(CLISubcommand):
16 + """ The `bench` subcommand for the vLLM CLI. """
17 +
18 + def __init__(self):
19 + self.name = "bench"
20 + super().__init__()
21 +
22 + @staticmethod
23 + def cmd(args: argparse.Namespace) -> None:
24 + args.dispatch_function(args)
25 +
26 + def validate(self, args: argparse.Namespace) -> None:
27 + if args.bench_type in self.cmds:
28 + self.cmds[args.bench_type].validate(args)
29 +
30 + def subparser_init(
31 + self,
32 + subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
33 + bench_parser = subparsers.add_parser(
34 + "bench",
35 + help="vLLM bench subcommand.",
36 + usage="vllm bench <bench_type> [options]")
37 + bench_subparsers = bench_parser.add_subparsers(required=True,
38 + dest="bench_type")
39 + self.cmds = {}
40 + for cmd_module in BENCHMARK_CMD_MODULES:
41 + new_cmds = cmd_module.cmd_init()
42 + for cmd in new_cmds:
43 + cmd.subparser_init(bench_subparsers).set_defaults(
44 + dispatch_function=cmd.cmd)
45 + self.cmds[cmd.name] = cmd
46 + return bench_parser
47 +
48 +
49 +def cmd_init() -> list[CLISubcommand]:
50 + return [BenchmarkSubcommand()]
vllm/entrypoints/cli/benchmark/serve.py new
+29
@@ -0,0 +1,29 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import argparse
3 +
4 +from vllm.benchmarks.serve import add_cli_args, main
5 +from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
6 +from vllm.entrypoints.cli.types import CLISubcommand
7 +
8 +
9 +class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
10 + """ The `serve` subcommand for vllm bench. """
11 +
12 + def __init__(self):
13 + self.name = "serve"
14 + super().__init__()
15 +
16 + @property
17 + def help(self) -> str:
18 + return "Benchmark the online serving throughput."
19 +
20 + def add_cli_args(self, parser: argparse.ArgumentParser) -> None:
21 + add_cli_args(parser)
22 +
23 + @staticmethod
24 + def cmd(args: argparse.Namespace) -> None:
25 + main(args)
26 +
27 +
28 +def cmd_init() -> list[CLISubcommand]:
29 + return [BenchmarkServingSubcommand()]
vllm/entrypoints/cli/main.py
+2
@@ -5,6 +5,7 @@ import os
5 import signal
6 import sys
7
8 +import vllm.entrypoints.cli.benchmark.main
9 import vllm.entrypoints.cli.openai
10 import vllm.entrypoints.cli.serve
11 import vllm.version
@@ -16,6 +17,7 @@ logger = init_logger(__name__)
17 CMD_MODULES = [
18 vllm.entrypoints.cli.openai,
19 vllm.entrypoints.cli.serve,
20 + vllm.entrypoints.cli.benchmark.main,
21 ]
22
23