@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6fb2788163

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[CI/Build][Doc] Fully deprecate old bench scripts for serving / throughput / latency (#24411)

Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com>

Ye (Charlotte) Qi committed Sep 9, 2025 at 03:02 UTC 6fb27881634d89c2e70e9e5fbad1b918c0d916cf

4 files changed +35 -2240

benchmarks/README.md

+2 -2

@@ -694,7 +694,7 @@ python -m vllm.entrypoints.openai.api_server \

694 Send requests with images:

695

696 ```bash

697 -python benchmarks/benchmark_serving.py \

697 +vllm bench serve \

698 --backend openai-chat \

699 --model Qwen/Qwen2.5-VL-7B-Instruct \

700 --dataset-name sharegpt \

@@ -721,7 +721,7 @@ python -m vllm.entrypoints.openai.api_server \

721 Send requests with videos:

722

723 ```bash

724 -python benchmarks/benchmark_serving.py \

724 +vllm bench serve \

725 --backend openai-chat \

726 --model Qwen/Qwen2.5-VL-7B-Instruct \

727 --dataset-name sharegpt \

benchmarks/benchmark_latency.py

+11 -185

@@ -1,191 +1,17 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Benchmark the latency of processing a single batch of requests."""

3 +import sys

5 -import argparse

6 -import dataclasses

7 -import json

8 -import os

9 -import time

10 -from typing import Any, Optional

11 -

12 -import numpy as np

13 -from tqdm import tqdm

14 -from typing_extensions import deprecated

15 -

16 -import vllm.envs as envs

17 -from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json

18 -from vllm import LLM, SamplingParams

19 -from vllm.engine.arg_utils import EngineArgs

20 -from vllm.inputs import PromptType

21 -from vllm.sampling_params import BeamSearchParams

22 -from vllm.utils import FlexibleArgumentParser

23 -

24 -

25 -def save_to_pytorch_benchmark_format(

26 - args: argparse.Namespace, results: dict[str, Any]

27 -) -> None:

28 - pt_records = convert_to_pytorch_benchmark_format(

29 - args=args,

30 - metrics={"latency": results["latencies"]},

31 - extra_info={k: results[k] for k in ["avg_latency", "percentiles"]},

32 - )

33 - if pt_records:

34 - pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"

35 - write_to_json(pt_file, pt_records)

36 -

37 -

38 -@deprecated(

39 - "benchmark_latency.py is deprecated and will be removed in a "

40 - "future version. Please use 'vllm bench latency' instead.",

41 -)

42 -def main(args: argparse.Namespace):

43 - print(args)

44 -

45 - engine_args = EngineArgs.from_cli_args(args)

46 -

47 - # NOTE(woosuk): If the request cannot be processed in a single batch,

48 - # the engine will automatically process the request in multiple batches.

49 - llm = LLM(**dataclasses.asdict(engine_args))

50 - assert llm.llm_engine.model_config.max_model_len >= (

51 - args.input_len + args.output_len

52 - ), (

53 - "Please ensure that max_model_len is greater than"

54 - " the sum of input_len and output_len."

55 - )

56 -

57 - sampling_params = SamplingParams(

58 - n=args.n,

59 - temperature=1.0,

60 - top_p=1.0,

61 - ignore_eos=True,

62 - max_tokens=args.output_len,

63 - detokenize=not args.disable_detokenize,

64 - )

65 - print(sampling_params)

66 - dummy_prompt_token_ids = np.random.randint(

67 - 10000, size=(args.batch_size, args.input_len)

68 - )

69 - dummy_prompts: list[PromptType] = [

70 - {"prompt_token_ids": batch} for batch in dummy_prompt_token_ids.tolist()

71 - ]

72 -

73 - def llm_generate():

74 - if not args.use_beam_search:

75 - llm.generate(dummy_prompts, sampling_params=sampling_params, use_tqdm=False)

76 - else:

77 - llm.beam_search(

78 - dummy_prompts,

79 - BeamSearchParams(

80 - beam_width=args.n,

81 - max_tokens=args.output_len,

82 - ignore_eos=True,

83 - ),

84 - )

85 -

86 - def run_to_completion(profile_dir: Optional[str] = None):

87 - if profile_dir:

88 - llm.start_profile()

89 - llm_generate()

90 - llm.stop_profile()

91 - else:

92 - start_time = time.perf_counter()

93 - llm_generate()

94 - end_time = time.perf_counter()

95 - latency = end_time - start_time

96 - return latency

97 -

98 - print("Warming up...")

99 - for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):

100 - run_to_completion(profile_dir=None)

101 -

102 - if args.profile:

103 - profile_dir = envs.VLLM_TORCH_PROFILER_DIR

104 - print(f"Profiling (results will be saved to '{profile_dir}')...")

105 - run_to_completion(profile_dir=profile_dir)

106 - return

107 -

108 - # Benchmark.

109 - latencies = []

110 - for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):

111 - latencies.append(run_to_completion(profile_dir=None))

112 - latencies = np.array(latencies)

113 - percentages = [10, 25, 50, 75, 90, 99]

114 - percentiles = np.percentile(latencies, percentages)

115 - print(f"Avg latency: {np.mean(latencies)} seconds")

116 - for percentage, percentile in zip(percentages, percentiles):

117 - print(f"{percentage}% percentile latency: {percentile} seconds")

118 -

119 - # Output JSON results if specified

120 - if args.output_json:

121 - results = {

122 - "avg_latency": np.mean(latencies),

123 - "latencies": latencies.tolist(),

124 - "percentiles": dict(zip(percentages, percentiles.tolist())),

125 - }

126 - with open(args.output_json, "w") as f:

127 - json.dump(results, f, indent=4)

128 - save_to_pytorch_benchmark_format(args, results)

129 -

130 -

131 -def create_argument_parser():

132 - parser = FlexibleArgumentParser(

133 - description="Benchmark the latency of processing a single batch of "

134 - "requests till completion."

135 - )

136 - parser.add_argument("--input-len", type=int, default=32)

137 - parser.add_argument("--output-len", type=int, default=128)

138 - parser.add_argument("--batch-size", type=int, default=8)

139 - parser.add_argument(

140 - "--n",

141 - type=int,

142 - default=1,

143 - help="Number of generated sequences per prompt.",

144 - )

145 - parser.add_argument("--use-beam-search", action="store_true")

146 - parser.add_argument(

147 - "--num-iters-warmup",

148 - type=int,

149 - default=10,

150 - help="Number of iterations to run for warmup.",

151 - )

152 - parser.add_argument(

153 - "--num-iters", type=int, default=30, help="Number of iterations to run."

154 - )

155 - parser.add_argument(

156 - "--profile",

157 - action="store_true",

158 - help="profile the generation process of a single batch",

159 - )

160 - parser.add_argument(

161 - "--output-json",

162 - type=str,

163 - default=None,

164 - help="Path to save the latency results in JSON format.",

165 - )

166 - parser.add_argument(

167 - "--disable-detokenize",

168 - action="store_true",

169 - help=(

170 - "Do not detokenize responses (i.e. do not include "

171 - "detokenization time in the latency measurement)"

172 - ),

173 - )

174 -

175 - parser = EngineArgs.add_cli_args(parser)

176 - # V1 enables prefix caching by default which skews the latency

177 - # numbers. We need to disable prefix caching by default.

178 - parser.set_defaults(enable_prefix_caching=False)

5 +if __name__ == "__main__":

6 + print("""DEPRECATED: This script has been moved to the vLLM CLI.

180 - return parser

8 +Please use the following command instead:

9 + vllm bench latency

11 +For help with the new command, run:

12 + vllm bench latency --help

183 -if __name__ == "__main__":

184 - parser = create_argument_parser()

185 - args = parser.parse_args()

186 - if args.profile and not envs.VLLM_TORCH_PROFILER_DIR:

187 - raise OSError(

188 - "The environment variable 'VLLM_TORCH_PROFILER_DIR' is not set. "

189 - "Please set it to a valid path to use torch profiler."

190 - )

191 - main(args)

14 +Alternatively, you can run the new command directly with:

15 + python -m vllm.entrypoints.cli.main bench latency --help

16 +""")

17 + sys.exit(1)

benchmarks/benchmark_serving.py

+11 -1318

@@ -1,1324 +1,17 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -r"""Benchmark online serving throughput.

3 +import sys

5 -On the server side, run one of the following commands:

6 - vLLM OpenAI API server

7 - vllm serve <your_model> \

8 - --swap-space 16

9 -

10 -On the client side, run:

11 - python benchmarks/benchmark_serving.py \

12 - --backend <backend> \

13 - --model <your_model> \

14 - --dataset-name sharegpt \

15 - --dataset-path <path to dataset> \

16 - --request-rate <request_rate> \ # By default <request_rate> is inf

17 - --num-prompts <num_prompts> # By default <num_prompts> is 1000

18 -

19 - when using tgi backend, add

20 - --endpoint /generate_stream

21 - to the end of the command above.

22 -"""

23 -

24 -import argparse

25 -import asyncio

26 -import gc

27 -import json

28 -import os

29 -import random

30 -import time

31 -import warnings

32 -from collections.abc import Iterable

33 -from dataclasses import dataclass

34 -from datetime import datetime

35 -from typing import Any, Literal, Optional

36 -

37 -import numpy as np

38 -from tqdm.asyncio import tqdm

39 -from transformers import PreTrainedTokenizerBase

40 -from typing_extensions import deprecated

41 -

42 -from backend_request_func import (

43 - ASYNC_REQUEST_FUNCS,

44 - OPENAI_COMPATIBLE_BACKENDS,

45 - RequestFuncInput,

46 - RequestFuncOutput,

47 -)

48 -

49 -try:

50 - from vllm.transformers_utils.tokenizer import get_tokenizer

51 -except ImportError:

52 - from backend_request_func import get_tokenizer

53 -

54 -try:

55 - from vllm.utils import FlexibleArgumentParser

56 -except ImportError:

57 - from argparse import ArgumentParser as FlexibleArgumentParser

58 -

59 -from benchmark_dataset import (

60 - AIMODataset,

61 - ASRDataset,

62 - BurstGPTDataset,

63 - ConversationDataset,

64 - CustomDataset,

65 - HuggingFaceDataset,

66 - InstructCoderDataset,

67 - MTBenchDataset,

68 - NextEditPredictionDataset,

69 - RandomDataset,

70 - SampleRequest,

71 - ShareGPTDataset,

72 - SonnetDataset,

73 - VisionArenaDataset,

74 -)

75 -from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json

76 -from vllm.benchmarks.serve import get_request

77 -

78 -MILLISECONDS_TO_SECONDS_CONVERSION = 1000

79 -

80 -

81 -@dataclass

82 -class BenchmarkMetrics:

83 - completed: int

84 - total_input: int

85 - total_output: int

86 - request_throughput: float

87 - request_goodput: float

88 - output_throughput: float

89 - total_token_throughput: float

90 - mean_ttft_ms: float

91 - median_ttft_ms: float

92 - std_ttft_ms: float

93 - percentiles_ttft_ms: list[tuple[float, float]]

94 - mean_tpot_ms: float

95 - median_tpot_ms: float

96 - std_tpot_ms: float

97 - percentiles_tpot_ms: list[tuple[float, float]]

98 - mean_itl_ms: float

99 - median_itl_ms: float

100 - std_itl_ms: float

101 - percentiles_itl_ms: list[tuple[float, float]]

102 - # E2EL stands for end-to-end latency per request.

103 - # It is the time taken on the client side from sending

104 - # a request to receiving a complete response.

105 - mean_e2el_ms: float

106 - median_e2el_ms: float

107 - std_e2el_ms: float

108 - percentiles_e2el_ms: list[tuple[float, float]]

109 -

110 -

111 -def calculate_metrics(

112 - input_requests: list[SampleRequest],

113 - outputs: list[RequestFuncOutput],

114 - dur_s: float,

115 - tokenizer: PreTrainedTokenizerBase,

116 - selected_percentile_metrics: list[str],

117 - selected_percentiles: list[float],

118 - goodput_config_dict: dict[str, float],

119 -) -> tuple[BenchmarkMetrics, list[int]]:

120 - actual_output_lens: list[int] = []

121 - total_input = 0

122 - completed = 0

123 - good_completed = 0

124 - itls: list[float] = []

125 - tpots: list[float] = []

126 - all_tpots: list[float] = []

127 - ttfts: list[float] = []

128 - e2els: list[float] = []

129 - for i in range(len(outputs)):

130 - if outputs[i].success:

131 - output_len = outputs[i].output_tokens

132 -

133 - if not output_len:

134 - # We use the tokenizer to count the number of output tokens

135 - # for some serving backends instead of looking at

136 - # len(outputs[i].itl) since multiple output tokens may be

137 - # bundled together

138 - # Note : this may inflate the output token count slightly

139 - output_len = len(

140 - tokenizer(

141 - outputs[i].generated_text, add_special_tokens=False

142 - ).input_ids

143 - )

144 - actual_output_lens.append(output_len)

145 - total_input += input_requests[i].prompt_len

146 - tpot = 0

147 - if output_len > 1:

148 - latency_minus_ttft = outputs[i].latency - outputs[i].ttft

149 - tpot = latency_minus_ttft / (output_len - 1)

150 - tpots.append(tpot)

151 - # Note: if output_len <= 1, we regard tpot as 0 for goodput

152 - all_tpots.append(tpot)

153 - itls += outputs[i].itl

154 - ttfts.append(outputs[i].ttft)

155 - e2els.append(outputs[i].latency)

156 - completed += 1

157 - else:

158 - actual_output_lens.append(0)

159 -

160 - if goodput_config_dict:

161 - valid_metrics = []

162 - slo_values = []

163 -

164 - if "ttft" in goodput_config_dict:

165 - valid_metrics.append(ttfts)

166 - slo_values.append(

167 - goodput_config_dict["ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION

168 - )

169 - if "tpot" in goodput_config_dict:

170 - valid_metrics.append(all_tpots)

171 - slo_values.append(

172 - goodput_config_dict["tpot"] / MILLISECONDS_TO_SECONDS_CONVERSION

173 - )

174 - if "e2el" in goodput_config_dict:

175 - valid_metrics.append(e2els)

176 - slo_values.append(

177 - goodput_config_dict["e2el"] / MILLISECONDS_TO_SECONDS_CONVERSION

178 - )

179 -

180 - for req_metric in zip(*valid_metrics):

181 - is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])

182 - if is_good_req:

183 - good_completed += 1

184 -

185 - if completed == 0:

186 - warnings.warn(

187 - "All requests failed. This is likely due to a misconfiguration "

188 - "on the benchmark arguments.",

189 - stacklevel=2,

190 - )

191 - metrics = BenchmarkMetrics(

192 - completed=completed,

193 - total_input=total_input,

194 - total_output=sum(actual_output_lens),

195 - request_throughput=completed / dur_s,

196 - request_goodput=good_completed / dur_s,

197 - output_throughput=sum(actual_output_lens) / dur_s,

198 - total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,

199 - mean_ttft_ms=np.mean(ttfts or 0)

200 - * 1000, # ttfts is empty if streaming is not supported by backend

201 - std_ttft_ms=np.std(ttfts or 0) * 1000,

202 - median_ttft_ms=np.median(ttfts or 0) * 1000,

203 - percentiles_ttft_ms=[

204 - (p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles

205 - ],

206 - mean_tpot_ms=np.mean(tpots or 0) * 1000,

207 - std_tpot_ms=np.std(tpots or 0) * 1000,

208 - median_tpot_ms=np.median(tpots or 0) * 1000,

209 - percentiles_tpot_ms=[

210 - (p, np.percentile(tpots or 0, p) * 1000) for p in selected_percentiles

211 - ],

212 - mean_itl_ms=np.mean(itls or 0) * 1000,

213 - std_itl_ms=np.std(itls or 0) * 1000,

214 - median_itl_ms=np.median(itls or 0) * 1000,

215 - percentiles_itl_ms=[

216 - (p, np.percentile(itls or 0, p) * 1000) for p in selected_percentiles

217 - ],

218 - mean_e2el_ms=np.mean(e2els or 0) * 1000,

219 - std_e2el_ms=np.std(e2els or 0) * 1000,

220 - median_e2el_ms=np.median(e2els or 0) * 1000,

221 - percentiles_e2el_ms=[

222 - (p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles

223 - ],

224 - )

225 -

226 - return metrics, actual_output_lens

227 -

228 -

229 -async def benchmark(

230 - backend: str,

231 - api_url: str,

232 - base_url: str,

233 - model_id: str,

234 - model_name: str,

235 - tokenizer: PreTrainedTokenizerBase,

236 - input_requests: list[SampleRequest],

237 - logprobs: Optional[int],

238 - request_rate: float,

239 - burstiness: float,

240 - disable_tqdm: bool,

241 - profile: bool,

242 - selected_percentile_metrics: list[str],

243 - selected_percentiles: list[float],

244 - ignore_eos: bool,

245 - goodput_config_dict: dict[str, float],

246 - max_concurrency: Optional[int],

247 - lora_modules: Optional[Iterable[str]],

248 - extra_body: Optional[dict],

249 - ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,

250 - ramp_up_start_rps: Optional[int] = None,

251 - ramp_up_end_rps: Optional[int] = None,

252 -):

253 - if backend in ASYNC_REQUEST_FUNCS:

254 - request_func = ASYNC_REQUEST_FUNCS[backend]

255 - else:

256 - raise ValueError(f"Unknown backend: {backend}")

257 -

258 - print("Starting initial single prompt test run...")

259 - test_prompt, test_prompt_len, test_output_len, test_mm_content = (

260 - input_requests[0].prompt,

261 - input_requests[0].prompt_len,

262 - input_requests[0].expected_output_len,

263 - input_requests[0].multi_modal_data,

264 - )

265 -

266 - assert (

267 - test_mm_content is None

268 - or isinstance(test_mm_content, dict)

269 - or (

270 - isinstance(test_mm_content, list)

271 - and all(isinstance(item, dict) for item in test_mm_content)

272 - )

273 - ), "multi_modal_data must be a dict or list[dict]"

274 - test_input = RequestFuncInput(

275 - model=model_id,

276 - model_name=model_name,

277 - prompt=test_prompt,

278 - api_url=api_url,

279 - prompt_len=test_prompt_len,

280 - output_len=test_output_len,

281 - logprobs=logprobs,

282 - multi_modal_content=test_mm_content,

283 - ignore_eos=ignore_eos,

284 - extra_body=extra_body,

285 - )

286 -

287 - test_output = await request_func(request_func_input=test_input)

288 - if not test_output.success:

289 - raise ValueError(

290 - "Initial test run failed - Please make sure benchmark arguments "

291 - f"are correctly specified. Error: {test_output.error}"

292 - )

293 - else:

294 - print("Initial test run completed. Starting main benchmark run...")

295 -

296 - if lora_modules:

297 - # For each input request, choose a LoRA module at random.

298 - lora_modules = iter(

299 - [random.choice(lora_modules) for _ in range(len(input_requests))]

300 - )

301 -

302 - if profile:

303 - print("Starting profiler...")

304 - profile_input = RequestFuncInput(

305 - model=model_id,

306 - model_name=model_name,

307 - prompt=test_prompt,

308 - api_url=base_url + "/start_profile",

309 - prompt_len=test_prompt_len,

310 - output_len=test_output_len,

311 - logprobs=logprobs,

312 - multi_modal_content=test_mm_content,

313 - ignore_eos=ignore_eos,

314 - extra_body=extra_body,

315 - )

316 - profile_output = await request_func(request_func_input=profile_input)

317 - if profile_output.success:

318 - print("Profiler started")

319 -

320 - distribution = "Poisson process" if burstiness == 1.0 else "Gamma distribution"

321 -

322 - if ramp_up_strategy is not None:

323 - print(

324 - f"Traffic ramp-up strategy: {ramp_up_strategy}. Will increase "

325 - f"RPS from {ramp_up_start_rps} to {ramp_up_end_rps} RPS over "

326 - "the duration of the benchmark."

327 - )

328 - else:

329 - print(f"Traffic request rate: {request_rate} RPS.")

330 -

331 - print(f"Burstiness factor: {burstiness} ({distribution})")

332 - print(f"Maximum request concurrency: {max_concurrency}")

333 -

334 - pbar = None if disable_tqdm else tqdm(total=len(input_requests))

335 -

336 - # This can be used once the minimum Python version is 3.10 or higher,

337 - # and it will simplify the code in limited_request_func.

338 - # semaphore = (asyncio.Semaphore(max_concurrency)

339 - # if max_concurrency else contextlib.nullcontext())

340 - semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None

341 -

342 - async def limited_request_func(request_func_input, pbar):

343 - if semaphore is None:

344 - return await request_func(request_func_input=request_func_input, pbar=pbar)

345 - async with semaphore:

346 - return await request_func(request_func_input=request_func_input, pbar=pbar)

347 -

348 - benchmark_start_time = time.perf_counter()

349 - tasks: list[asyncio.Task] = []

350 -

351 - rps_change_events = []

352 - last_int_rps = -1

353 - if ramp_up_strategy is not None and ramp_up_start_rps is not None:

354 - last_int_rps = ramp_up_start_rps

355 - rps_change_events.append(

356 - {

357 - "rps": last_int_rps,

358 - "timestamp": datetime.now().isoformat(),

359 - }

360 - )

361 -

362 - async for request, current_request_rate in get_request(

363 - input_requests,

364 - request_rate,

365 - burstiness,

366 - ramp_up_strategy,

367 - ramp_up_start_rps,

368 - ramp_up_end_rps,

369 - ):

370 - if ramp_up_strategy is not None:

371 - current_int_rps = int(current_request_rate)

372 - if current_int_rps > last_int_rps:

373 - timestamp = datetime.now().isoformat()

374 - for rps_val in range(last_int_rps + 1, current_int_rps + 1):

375 - rps_change_events.append({"rps": rps_val, "timestamp": timestamp})

376 - last_int_rps = current_int_rps

377 -

378 - prompt, prompt_len, output_len, mm_content, request_id = (

379 - request.prompt,

380 - request.prompt_len,

381 - request.expected_output_len,

382 - request.multi_modal_data,

383 - request.request_id,

384 - )

385 - req_model_id, req_model_name = model_id, model_name

386 - if lora_modules:

387 - req_lora_module = next(lora_modules)

388 - req_model_id, req_model_name = req_lora_module, req_lora_module

389 -

390 - request_func_input = RequestFuncInput(

391 - model=req_model_id,

392 - model_name=req_model_name,

393 - prompt=prompt,

394 - api_url=api_url,

395 - prompt_len=prompt_len,

396 - output_len=output_len,

397 - logprobs=logprobs,

398 - multi_modal_content=mm_content,

399 - ignore_eos=ignore_eos,

400 - extra_body=extra_body,

401 - request_id=request_id,

402 - )

403 - task = limited_request_func(request_func_input=request_func_input, pbar=pbar)

404 - tasks.append(asyncio.create_task(task))

405 - outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)

406 -

407 - if pbar is not None:

408 - pbar.close()

409 -

410 - benchmark_duration = time.perf_counter() - benchmark_start_time

411 -

412 - metrics, actual_output_lens = calculate_metrics(

413 - input_requests=input_requests,

414 - outputs=outputs,

415 - dur_s=benchmark_duration,

416 - tokenizer=tokenizer,

417 - selected_percentile_metrics=selected_percentile_metrics,

418 - selected_percentiles=selected_percentiles,

419 - goodput_config_dict=goodput_config_dict,

420 - )

421 -

422 - print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))

423 - print("{:<40} {:<10}".format("Successful requests:", metrics.completed))

424 - if max_concurrency is not None:

425 - print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency))

426 - if request_rate != float("inf"):

427 - print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate))

428 - print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))

429 - print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))

430 - print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))

431 - print(

432 - "{:<40} {:<10.2f}".format(

433 - "Request throughput (req/s):", metrics.request_throughput

434 - )

435 - )

436 - if goodput_config_dict:

437 - print(

438 - "{:<40} {:<10.2f}".format(

439 - "Request goodput (req/s):", metrics.request_goodput

440 - )

441 - )

442 - print(

443 - "{:<40} {:<10.2f}".format(

444 - "Output token throughput (tok/s):", metrics.output_throughput

445 - )

446 - )

447 - print(

448 - "{:<40} {:<10.2f}".format(

449 - "Total Token throughput (tok/s):", metrics.total_token_throughput

450 - )

451 - )

452 -

453 - result = {

454 - "duration": benchmark_duration,

455 - "completed": metrics.completed,

456 - "total_input_tokens": metrics.total_input,

457 - "total_output_tokens": metrics.total_output,

458 - "request_throughput": metrics.request_throughput,

459 - "request_goodput": metrics.request_goodput if goodput_config_dict else None,

460 - "output_throughput": metrics.output_throughput,

461 - "total_token_throughput": metrics.total_token_throughput,

462 - "input_lens": [output.prompt_len for output in outputs],

463 - "output_lens": actual_output_lens,

464 - "ttfts": [output.ttft for output in outputs],

465 - "itls": [output.itl for output in outputs],

466 - "generated_texts": [output.generated_text for output in outputs],

467 - "errors": [output.error for output in outputs],

468 - }

469 -

470 - if rps_change_events:

471 - result["rps_change_events"] = rps_change_events

472 -

473 - def process_one_metric(

474 - # E.g., "ttft"

475 - metric_attribute_name: str,

476 - # E.g., "TTFT"

477 - metric_name: str,

478 - # E.g., "Time to First Token"

479 - metric_header: str,

480 - ):

481 - # This function prints and adds statistics of the specified

482 - # metric.

483 - if metric_attribute_name not in selected_percentile_metrics:

484 - return

485 - print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-"))

486 - print(

487 - "{:<40} {:<10.2f}".format(

488 - f"Mean {metric_name} (ms):",

489 - getattr(metrics, f"mean_{metric_attribute_name}_ms"),

490 - )

491 - )

492 - print(

493 - "{:<40} {:<10.2f}".format(

494 - f"Median {metric_name} (ms):",

495 - getattr(metrics, f"median_{metric_attribute_name}_ms"),

496 - )

497 - )

498 - result[f"mean_{metric_attribute_name}_ms"] = getattr(

499 - metrics, f"mean_{metric_attribute_name}_ms"

500 - )

501 - result[f"median_{metric_attribute_name}_ms"] = getattr(

502 - metrics, f"median_{metric_attribute_name}_ms"

503 - )

504 - result[f"std_{metric_attribute_name}_ms"] = getattr(

505 - metrics, f"std_{metric_attribute_name}_ms"

506 - )

507 - for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"):

508 - p_word = str(int(p)) if int(p) == p else str(p)

509 - print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value))

510 - result[f"p{p_word}_{metric_attribute_name}_ms"] = value

511 -

512 - process_one_metric("ttft", "TTFT", "Time to First Token")

513 - process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)")

514 - process_one_metric("itl", "ITL", "Inter-token Latency")

515 - process_one_metric("e2el", "E2EL", "End-to-end Latency")

516 -

517 - print("=" * 50)

518 -

519 - if profile:

520 - print("Stopping profiler...")

521 - profile_input = RequestFuncInput(

522 - model=model_id,

523 - prompt=test_prompt,

524 - api_url=base_url + "/stop_profile",

525 - prompt_len=test_prompt_len,

526 - output_len=test_output_len,

527 - logprobs=logprobs,

528 - )

529 - profile_output = await request_func(request_func_input=profile_input)

530 - if profile_output.success:

531 - print("Profiler stopped")

532 -

533 - return result

534 -

535 -

536 -def check_goodput_args(args):

537 - # Check and parse goodput arguments

538 - goodput_config_dict = {}

539 - VALID_NAMES = ["ttft", "tpot", "e2el"]

540 - if args.goodput:

541 - goodput_config_dict = parse_goodput(args.goodput)

542 - for slo_name, slo_val in goodput_config_dict.items():

543 - if slo_name not in VALID_NAMES:

544 - raise ValueError(

545 - f"Invalid metric name found, {slo_name}: {slo_val}. "

546 - "The service level objective name should be one of "

547 - f"{str(VALID_NAMES)}. "

548 - )

549 - if slo_val < 0:

550 - raise ValueError(

551 - f"Invalid value found, {slo_name}: {slo_val}. "

552 - "The service level objective value should be "

553 - "non-negative."

554 - )

555 - return goodput_config_dict

556 -

557 -

558 -def parse_goodput(slo_pairs):

559 - goodput_config_dict = {}

560 - try:

561 - for slo_pair in slo_pairs:

562 - slo_name, slo_val = slo_pair.split(":")

563 - goodput_config_dict[slo_name] = float(slo_val)

564 - except ValueError as err:

565 - raise argparse.ArgumentTypeError(

566 - "Invalid format found for service level objectives. "

567 - 'Specify service level objectives for goodput as "KEY:VALUE" '

568 - "pairs, where the key is a metric name, and the value is a "

569 - "number in milliseconds."

570 - ) from err

571 - return goodput_config_dict

572 -

573 -

574 -def save_to_pytorch_benchmark_format(

575 - args: argparse.Namespace, results: dict[str, Any], file_name: str

576 -) -> None:

577 - metrics = [

578 - "median_ttft_ms",

579 - "mean_ttft_ms",

580 - "std_ttft_ms",

581 - "p99_ttft_ms",

582 - "mean_tpot_ms",

583 - "median_tpot_ms",

584 - "std_tpot_ms",

585 - "p99_tpot_ms",

586 - "median_itl_ms",

587 - "mean_itl_ms",

588 - "std_itl_ms",

589 - "p99_itl_ms",

590 - ]

591 - # These raw data might be useful, but they are rather big. They can be added

592 - # later if needed

593 - ignored_metrics = ["ttfts", "itls", "generated_texts", "errors"]

594 - pt_records = convert_to_pytorch_benchmark_format(

595 - args=args,

596 - metrics={k: [results[k]] for k in metrics},

597 - extra_info={

598 - k: results[k]

599 - for k in results

600 - if k not in metrics and k not in ignored_metrics

601 - },

602 - )

603 - if pt_records:

604 - # Don't use json suffix here as we don't want CI to pick it up

605 - pt_file = f"{os.path.splitext(file_name)[0]}.pytorch.json"

606 - write_to_json(pt_file, pt_records)

607 -

608 -

609 -@deprecated(

610 - "benchmark_serving.py is deprecated and will be removed in a future "

611 - "version. Please use 'vllm bench serve' instead.",

612 -)

613 -def main(args: argparse.Namespace):

614 - print(args)

615 - random.seed(args.seed)

616 - np.random.seed(args.seed)

617 -

618 - backend = args.backend

619 - model_id = args.model

620 - model_name = args.served_model_name

621 - tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model

622 - tokenizer_mode = args.tokenizer_mode

623 -

624 - # Validate ramp-up arguments

625 - if args.ramp_up_strategy is not None:

626 - if args.request_rate != float("inf"):

627 - raise ValueError(

628 - "When using ramp-up, do not specify --request-rate. "

629 - "The request rate will be controlled by ramp-up parameters. "

630 - "Please remove the --request-rate argument."

631 - )

632 - if args.ramp_up_start_rps is None or args.ramp_up_end_rps is None:

633 - raise ValueError(

634 - "When using --ramp-up-strategy, both --ramp-up-start-rps and "

635 - "--ramp-up-end-rps must be specified"

636 - )

637 - if args.ramp_up_start_rps < 0 or args.ramp_up_end_rps < 0:

638 - raise ValueError("Ramp-up start and end RPS must be non-negative")

639 - if args.ramp_up_start_rps > args.ramp_up_end_rps:

640 - raise ValueError("Ramp-up start RPS must be less than end RPS")

641 - if args.ramp_up_strategy == "exponential" and args.ramp_up_start_rps == 0:

642 - raise ValueError("For exponential ramp-up, the start RPS cannot be 0.")

643 -

644 - if args.base_url is not None:

645 - api_url = f"{args.base_url}{args.endpoint}"

646 - base_url = f"{args.base_url}"

647 - else:

648 - api_url = f"http://{args.host}:{args.port}{args.endpoint}"

649 - base_url = f"http://{args.host}:{args.port}"

650 -

651 - tokenizer = get_tokenizer(

652 - tokenizer_id,

653 - tokenizer_mode=tokenizer_mode,

654 - trust_remote_code=args.trust_remote_code,

655 - )

656 -

657 - if args.dataset_name is None:

658 - raise ValueError(

659 - "Please specify '--dataset-name' and the corresponding "

660 - "'--dataset-path' if required."

661 - )

662 -

663 - if args.dataset_name == "custom":

664 - dataset = CustomDataset(dataset_path=args.dataset_path)

665 - input_requests = dataset.sample(

666 - num_requests=args.num_prompts,

667 - tokenizer=tokenizer,

668 - output_len=args.custom_output_len,

669 - skip_chat_template=args.custom_skip_chat_template,

670 - request_id_prefix=args.request_id_prefix,

671 - )

672 -

673 - elif args.dataset_name == "sonnet":

674 - dataset = SonnetDataset(dataset_path=args.dataset_path)

675 - # For the "sonnet" dataset, formatting depends on the backend.

676 - if args.backend == "openai-chat":

677 - input_requests = dataset.sample(

678 - num_requests=args.num_prompts,

679 - input_len=args.sonnet_input_len,

680 - output_len=args.sonnet_output_len,

681 - prefix_len=args.sonnet_prefix_len,

682 - tokenizer=tokenizer,

683 - return_prompt_formatted=False,

684 - request_id_prefix=args.request_id_prefix,

685 - )

686 - else:

687 - assert tokenizer.chat_template or tokenizer.default_chat_template, (

688 - "Tokenizer/model must have chat template for sonnet dataset."

689 - )

690 - input_requests = dataset.sample(

691 - num_requests=args.num_prompts,

692 - input_len=args.sonnet_input_len,

693 - output_len=args.sonnet_output_len,

694 - prefix_len=args.sonnet_prefix_len,

695 - tokenizer=tokenizer,

696 - return_prompt_formatted=True,

697 - request_id_prefix=args.request_id_prefix,

698 - )

699 -

700 - elif args.dataset_name == "hf":

701 - # all following datasets are implemented from the

702 - # HuggingFaceDataset base class

703 - if args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS:

704 - dataset_class = VisionArenaDataset

705 - args.hf_split = "train"

706 - args.hf_subset = None

707 - elif args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS:

708 - dataset_class = InstructCoderDataset

709 - args.hf_split = "train"

710 - elif args.dataset_path in MTBenchDataset.SUPPORTED_DATASET_PATHS:

711 - dataset_class = MTBenchDataset

712 - args.hf_split = "train"

713 - elif args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS:

714 - dataset_class = ConversationDataset

715 - elif args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS:

716 - dataset_class = AIMODataset

717 - args.hf_split = "train"

718 - elif args.dataset_path in NextEditPredictionDataset.SUPPORTED_DATASET_PATHS: # noqa: E501

719 - dataset_class = NextEditPredictionDataset

720 - args.hf_split = "train"

721 - elif args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS:

722 - dataset_class = ASRDataset

723 - args.hf_split = "train"

724 - else:

725 - supported_datasets = set(

726 - [

727 - dataset_name

728 - for cls in HuggingFaceDataset.__subclasses__()

729 - for dataset_name in cls.SUPPORTED_DATASET_PATHS

730 - ]

731 - )

732 - raise ValueError(

733 - f"Unsupported dataset path: {args.dataset_path}. "

734 - "Huggingface dataset only supports dataset_path"

735 - f" from one of following: {supported_datasets}. "

736 - "Please consider contributing if you would "

737 - "like to add support for additional dataset formats."

738 - )

739 -

740 - if dataset_class.IS_MULTIMODAL and backend not in [

741 - "openai-chat",

742 - "openai-audio",

743 - ]:

744 - # multi-modal benchmark is only available on OpenAI Chat backend.

745 - raise ValueError(

746 - "Multi-modal content is only supported on 'openai-chat' and "

747 - "'openai-audio' backend."

748 - )

749 - input_requests = dataset_class(

750 - dataset_path=args.dataset_path,

751 - dataset_subset=args.hf_subset,

752 - dataset_split=args.hf_split,

753 - random_seed=args.seed,

754 - no_stream=args.no_stream,

755 - ).sample(

756 - num_requests=args.num_prompts,

757 - tokenizer=tokenizer,

758 - output_len=args.hf_output_len,

759 - request_id_prefix=args.request_id_prefix,

760 - )

761 -

762 - else:

763 - # For datasets that follow a similar structure, use a mapping.

764 - dataset_mapping = {

765 - "sharegpt": lambda: ShareGPTDataset(

766 - random_seed=args.seed, dataset_path=args.dataset_path

767 - ).sample(

768 - tokenizer=tokenizer,

769 - num_requests=args.num_prompts,

770 - output_len=args.sharegpt_output_len,

771 - request_id_prefix=args.request_id_prefix,

772 - ),

773 - "burstgpt": lambda: BurstGPTDataset(

774 - random_seed=args.seed, dataset_path=args.dataset_path

775 - ).sample(

776 - tokenizer=tokenizer,

777 - num_requests=args.num_prompts,

778 - request_id_prefix=args.request_id_prefix,

779 - ),

780 - "random": lambda: RandomDataset(dataset_path=args.dataset_path).sample(

781 - tokenizer=tokenizer,

782 - num_requests=args.num_prompts,

783 - prefix_len=args.random_prefix_len,

784 - input_len=args.random_input_len,

785 - output_len=args.random_output_len,

786 - range_ratio=args.random_range_ratio,

787 - request_id_prefix=args.request_id_prefix,

788 - ),

789 - }

790 -

791 - try:

792 - input_requests = dataset_mapping[args.dataset_name]()

793 - except KeyError as err:

794 - raise ValueError(f"Unknown dataset: {args.dataset_name}") from err

795 - goodput_config_dict = check_goodput_args(args)

796 -

797 - # Collect the sampling parameters.

798 - sampling_params = {

799 - k: v

800 - for k, v in {

801 - "top_p": args.top_p,

802 - "top_k": args.top_k,

803 - "min_p": args.min_p,

804 - "temperature": args.temperature,

805 - }.items()

806 - if v is not None

807 - }

808 -

809 - # Sampling parameters are only supported by openai-compatible backend.

810 - if sampling_params and args.backend not in OPENAI_COMPATIBLE_BACKENDS:

811 - raise ValueError(

812 - "Sampling parameters are only supported by openai-compatible backends."

813 - )

814 -

815 - if "temperature" not in sampling_params:

816 - sampling_params["temperature"] = 0.0 # Default to greedy decoding.

817 -

818 - if args.backend == "llama.cpp":

819 - # Disable prompt caching in llama.cpp backend

820 - sampling_params["cache_prompt"] = False

821 -

822 - # Avoid GC processing "static" data - reduce pause times.

823 - gc.collect()

824 - gc.freeze()

825 -

826 - benchmark_result = asyncio.run(

827 - benchmark(

828 - backend=backend,

829 - api_url=api_url,

830 - base_url=base_url,

831 - model_id=model_id,

832 - model_name=model_name,

833 - tokenizer=tokenizer,

834 - input_requests=input_requests,

835 - logprobs=args.logprobs,

836 - request_rate=args.request_rate,

837 - burstiness=args.burstiness,

838 - disable_tqdm=args.disable_tqdm,

839 - profile=args.profile,

840 - selected_percentile_metrics=args.percentile_metrics.split(","),

841 - selected_percentiles=[float(p) for p in args.metric_percentiles.split(",")],

842 - ignore_eos=args.ignore_eos,

843 - goodput_config_dict=goodput_config_dict,

844 - max_concurrency=args.max_concurrency,

845 - lora_modules=args.lora_modules,

846 - extra_body=sampling_params,

847 - ramp_up_strategy=args.ramp_up_strategy,

848 - ramp_up_start_rps=args.ramp_up_start_rps,

849 - ramp_up_end_rps=args.ramp_up_end_rps,

850 - )

851 - )

852 -

853 - # Save config and results to json

854 - if args.save_result or args.append_result:

855 - result_json: dict[str, Any] = {}

856 -

857 - # Setup

858 - current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")

859 - result_json["date"] = current_dt

860 - result_json["backend"] = backend

861 - result_json["model_id"] = model_id

862 - result_json["tokenizer_id"] = tokenizer_id

863 - result_json["num_prompts"] = args.num_prompts

864 -

865 - # Metadata

866 - if args.metadata:

867 - for item in args.metadata:

868 - if "=" in item:

869 - kvstring = item.split("=")

870 - result_json[kvstring[0].strip()] = kvstring[1].strip()

871 - else:

872 - raise ValueError(

873 - "Invalid metadata format. Please use KEY=VALUE format."

874 - )

875 - # Traffic

876 - result_json["request_rate"] = (

877 - args.request_rate if args.request_rate < float("inf") else "inf"

878 - )

879 - result_json["burstiness"] = args.burstiness

880 - result_json["max_concurrency"] = args.max_concurrency

881 -

882 - if args.ramp_up_strategy is not None:

883 - result_json["ramp_up_strategy"] = args.ramp_up_strategy

884 - result_json["ramp_up_start_rps"] = args.ramp_up_start_rps

885 - result_json["ramp_up_end_rps"] = args.ramp_up_end_rps

886 -

887 - # Merge with benchmark result

888 - result_json = {**result_json, **benchmark_result}

889 -

890 - if not args.save_detailed:

891 - # Remove fields with too many data points

892 - for field in [

893 - "input_lens",

894 - "output_lens",

895 - "ttfts",

896 - "itls",

897 - "generated_texts",

898 - "errors",

899 - ]:

900 - if field in result_json:

901 - del result_json[field]

902 - if field in benchmark_result:

903 - del benchmark_result[field]

904 -

905 - # Save to file

906 - base_model_id = model_id.split("/")[-1]

907 - max_concurrency_str = (

908 - f"-concurrency{args.max_concurrency}"

909 - if args.max_concurrency is not None

910 - else ""

911 - )

912 - if args.ramp_up_strategy is not None:

913 - file_name = f"{backend}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa

914 - else:

915 - file_name = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa

916 - if args.result_filename:

917 - file_name = args.result_filename

918 - if args.result_dir:

919 - os.makedirs(args.result_dir, exist_ok=True)

920 - file_name = os.path.join(args.result_dir, file_name)

921 - with open(

922 - file_name, mode="a+" if args.append_result else "w", encoding="utf-8"

923 - ) as outfile:

924 - # Append a newline.

925 - if args.append_result and outfile.tell() != 0:

926 - outfile.write("\n")

927 - json.dump(result_json, outfile)

928 - save_to_pytorch_benchmark_format(args, result_json, file_name)

929 -

930 -

931 -def create_argument_parser():

932 - parser = FlexibleArgumentParser(

933 - description="Benchmark the online serving throughput."

934 - )

935 - parser.add_argument(

936 - "--backend",

937 - type=str,

938 - default="vllm",

939 - choices=list(ASYNC_REQUEST_FUNCS.keys()),

940 - )

941 - parser.add_argument(

942 - "--base-url",

943 - type=str,

944 - default=None,

945 - help="Server or API base url if not using http host and port.",

946 - )

947 - # Use 127.0.0.1 here instead of localhost to force the use of ipv4

948 - parser.add_argument("--host", type=str, default="127.0.0.1")

949 - parser.add_argument("--port", type=int, default=8000)

950 - parser.add_argument(

951 - "--endpoint",

952 - type=str,

953 - default="/v1/completions",

954 - help="API endpoint.",

955 - )

956 - parser.add_argument(

957 - "--dataset-name",

958 - type=str,

959 - default="sharegpt",

960 - choices=["sharegpt", "burstgpt", "sonnet", "random", "hf", "custom"],

961 - help="Name of the dataset to benchmark on.",

962 - )

963 - parser.add_argument(

964 - "--dataset-path",

965 - type=str,

966 - default=None,

967 - help="Path to the sharegpt/sonnet dataset. "

968 - "Or the huggingface dataset ID if using HF dataset.",

969 - )

970 - parser.add_argument(

971 - "--no-stream",

972 - action="store_true",

973 - help="Do not load the dataset in streaming mode.",

974 - )

975 - parser.add_argument(

976 - "--max-concurrency",

977 - type=int,

978 - default=None,

979 - help="Maximum number of concurrent requests. This can be used "

980 - "to help simulate an environment where a higher level component "

981 - "is enforcing a maximum number of concurrent requests. While the "

982 - "--request-rate argument controls the rate at which requests are "

983 - "initiated, this argument will control how many are actually allowed "

984 - "to execute at a time. This means that when used in combination, the "

985 - "actual request rate may be lower than specified with --request-rate, "

986 - "if the server is not processing requests fast enough to keep up.",

987 - )

988 -

989 - parser.add_argument(

990 - "--model",

991 - type=str,

992 - required=True,

993 - help="Name of the model.",

994 - )

995 - parser.add_argument(

996 - "--tokenizer",

997 - type=str,

998 - help="Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501

999 - )

1000 - parser.add_argument("--use-beam-search", action="store_true")

1001 - parser.add_argument(

1002 - "--num-prompts",

1003 - type=int,

1004 - default=1000,

1005 - help="Number of prompts to process.",

1006 - )

1007 - parser.add_argument(

1008 - "--logprobs",

1009 - type=int,

1010 - default=None,

1011 - help=(

1012 - "Number of logprobs-per-token to compute & return as part of "

1013 - "the request. If unspecified, then either (1) if beam search "

1014 - "is disabled, no logprobs are computed & a single dummy "

1015 - "logprob is returned for each token; or (2) if beam search "

1016 - "is enabled 1 logprob per token is computed"

1017 - ),

1018 - )

1019 - parser.add_argument(

1020 - "--request-rate",

1021 - type=float,

1022 - default=float("inf"),

1023 - help="Number of requests per second. If this is inf, "

1024 - "then all the requests are sent at time 0. "

1025 - "Otherwise, we use Poisson process or gamma distribution "

1026 - "to synthesize the request arrival times.",

1027 - )

1028 - parser.add_argument(

1029 - "--burstiness",

1030 - type=float,

1031 - default=1.0,

1032 - help="Burstiness factor of the request generation. "

1033 - "Only take effect when request_rate is not inf. "

1034 - "Default value is 1, which follows Poisson process. "

1035 - "Otherwise, the request intervals follow a gamma distribution. "

1036 - "A lower burstiness value (0 < burstiness < 1) results in more "

1037 - "bursty requests. A higher burstiness value (burstiness > 1) "

1038 - "results in a more uniform arrival of requests.",

1039 - )

1040 - parser.add_argument("--seed", type=int, default=0)

1041 - parser.add_argument(

1042 - "--trust-remote-code",

1043 - action="store_true",

1044 - help="Trust remote code from huggingface",

1045 - )

1046 - parser.add_argument(

1047 - "--disable-tqdm",

1048 - action="store_true",

1049 - help="Specify to disable tqdm progress bar.",

1050 - )

1051 - parser.add_argument(

1052 - "--profile",

1053 - action="store_true",

1054 - help="Use Torch Profiler. The endpoint must be launched with "

1055 - "VLLM_TORCH_PROFILER_DIR to enable profiler.",

1056 - )

1057 - parser.add_argument(

1058 - "--save-result",

1059 - action="store_true",

1060 - help="Specify to save benchmark results to a json file",

1061 - )

1062 - parser.add_argument(

1063 - "--save-detailed",

1064 - action="store_true",

1065 - help="When saving the results, whether to include per request "

1066 - "information such as response, error, ttfs, tpots, etc.",

1067 - )

1068 - parser.add_argument(

1069 - "--append-result",

1070 - action="store_true",

1071 - help="Append the benchmark result to the existing json file.",

1072 - )

1073 - parser.add_argument(

1074 - "--metadata",

1075 - metavar="KEY=VALUE",

1076 - nargs="*",

1077 - help="Key-value pairs (e.g, --metadata version=0.3.3 tp=1) "

1078 - "for metadata of this run to be saved in the result JSON file "

1079 - "for record keeping purposes.",

1080 - )

1081 - parser.add_argument(

1082 - "--result-dir",

1083 - type=str,

1084 - default=None,

1085 - help="Specify directory to save benchmark json results."

1086 - "If not specified, results are saved in the current directory.",

1087 - )

1088 - parser.add_argument(

1089 - "--result-filename",

1090 - type=str,

1091 - default=None,

1092 - help="Specify the filename to save benchmark json results."

1093 - "If not specified, results will be saved in "

1094 - "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"

1095 - " format.",

1096 - )

1097 - parser.add_argument(

1098 - "--ignore-eos",

1099 - action="store_true",

1100 - help="Set ignore_eos flag when sending the benchmark request."

1101 - "Warning: ignore_eos is not supported in deepspeed_mii and tgi.",

1102 - )

1103 - parser.add_argument(

1104 - "--percentile-metrics",

1105 - type=str,

1106 - default="ttft,tpot,itl",

1107 - help="Comma-separated list of selected metrics to report percentiles. "

1108 - "This argument specifies the metrics to report percentiles. "

1109 - 'Allowed metric names are "ttft", "tpot", "itl", "e2el". '

1110 - 'Default value is "ttft,tpot,itl".',

1111 - )

1112 - parser.add_argument(

1113 - "--metric-percentiles",

1114 - type=str,

1115 - default="99",

1116 - help="Comma-separated list of percentiles for selected metrics. "

1117 - 'To report 25-th, 50-th, and 75-th percentiles, use "25,50,75". '

1118 - 'Default value is "99". '

1119 - 'Use "--percentile-metrics" to select metrics.',

1120 - )

1121 - parser.add_argument(

1122 - "--goodput",

1123 - nargs="+",

1124 - required=False,

1125 - help='Specify service level objectives for goodput as "KEY:VALUE" '

1126 - "pairs, where the key is a metric name, and the value is in "

1127 - 'milliseconds. Multiple "KEY:VALUE" pairs can be provided, '

1128 - "separated by spaces. Allowed request level metric names are "

1129 - '"ttft", "tpot", "e2el". For more context on the definition of '

1130 - "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 "

1131 - "and the blog: https://hao-ai-lab.github.io/blogs/distserve",

1132 - )

1133 - parser.add_argument(

1134 - "--request-id-prefix",

1135 - type=str,

1136 - required=False,

1137 - default="benchmark-serving",

1138 - help="Specify the prefix of request id.",

1139 - )

1140 -

1141 - # group for dataset specific arguments

1142 - custom_group = parser.add_argument_group("custom dataset options")

1143 - custom_group.add_argument(

1144 - "--custom-output-len",

1145 - type=int,

1146 - default=256,

1147 - help="Number of output tokens per request, used only for custom dataset.",

1148 - )

1149 - custom_group.add_argument(

1150 - "--custom-skip-chat-template",

1151 - action="store_true",

1152 - help="Skip applying chat template to prompt, used only for custom dataset.",

1153 - )

1154 -

1155 - sonnet_group = parser.add_argument_group("sonnet dataset options")

1156 - sonnet_group.add_argument(

1157 - "--sonnet-input-len",

1158 - type=int,

1159 - default=550,

1160 - help="Number of input tokens per request, used only for sonnet dataset.",

1161 - )

1162 - sonnet_group.add_argument(

1163 - "--sonnet-output-len",

1164 - type=int,

1165 - default=150,

1166 - help="Number of output tokens per request, used only for sonnet dataset.",

1167 - )

1168 - sonnet_group.add_argument(

1169 - "--sonnet-prefix-len",

1170 - type=int,

1171 - default=200,

1172 - help="Number of prefix tokens per request, used only for sonnet dataset.",

1173 - )

1174 -

1175 - sharegpt_group = parser.add_argument_group("sharegpt dataset options")

1176 - sharegpt_group.add_argument(

1177 - "--sharegpt-output-len",

1178 - type=int,

1179 - default=None,

1180 - help="Output length for each request. Overrides the output length "

1181 - "from the ShareGPT dataset.",

1182 - )

1183 -

1184 - random_group = parser.add_argument_group("random dataset options")

1185 - random_group.add_argument(

1186 - "--random-input-len",

1187 - type=int,

1188 - default=1024,

1189 - help="Number of input tokens per request, used only for random sampling.",

1190 - )

1191 - random_group.add_argument(

1192 - "--random-output-len",

1193 - type=int,

1194 - default=128,

1195 - help="Number of output tokens per request, used only for random sampling.",

1196 - )

1197 - random_group.add_argument(

1198 - "--random-range-ratio",

1199 - type=float,

1200 - default=0.0,

1201 - help="Range ratio for sampling input/output length, "

1202 - "used only for random sampling. Must be in the range [0, 1) to define "

1203 - "a symmetric sampling range"

1204 - "[length * (1 - range_ratio), length * (1 + range_ratio)].",

1205 - )

1206 - random_group.add_argument(

1207 - "--random-prefix-len",

1208 - type=int,

1209 - default=0,

1210 - help=(

1211 - "Number of fixed prefix tokens before the random context "

1212 - "in a request. "

1213 - "The total input length is the sum of `random-prefix-len` and "

1214 - "a random "

1215 - "context length sampled from [input_len * (1 - range_ratio), "

1216 - "input_len * (1 + range_ratio)]."

1217 - ),

1218 - )

1219 -

1220 - hf_group = parser.add_argument_group("hf dataset options")

1221 - hf_group.add_argument(

1222 - "--hf-subset", type=str, default=None, help="Subset of the HF dataset."

1223 - )

1224 - hf_group.add_argument(

1225 - "--hf-split", type=str, default=None, help="Split of the HF dataset."

1226 - )

1227 - hf_group.add_argument(

1228 - "--hf-output-len",

1229 - type=int,

1230 - default=None,

1231 - help="Output length for each request. Overrides the output lengths "

1232 - "from the sampled HF dataset.",

1233 - )

1234 -

1235 - sampling_group = parser.add_argument_group("sampling parameters")

1236 - sampling_group.add_argument(

1237 - "--top-p",

1238 - type=float,

1239 - default=None,

1240 - help="Top-p sampling parameter. Only has effect on openai-compatible backends.",

1241 - )

1242 - sampling_group.add_argument(

1243 - "--top-k",

1244 - type=int,

1245 - default=None,

1246 - help="Top-k sampling parameter. Only has effect on openai-compatible backends.",

1247 - )

1248 - sampling_group.add_argument(

1249 - "--min-p",

1250 - type=float,

1251 - default=None,

1252 - help="Min-p sampling parameter. Only has effect on openai-compatible backends.",

1253 - )

1254 - sampling_group.add_argument(

1255 - "--temperature",

1256 - type=float,

1257 - default=None,

1258 - help="Temperature sampling parameter. Only has effect on "

1259 - "openai-compatible backends. If not specified, default to greedy "

1260 - "decoding (i.e. temperature==0.0).",

1261 - )

1262 -

1263 - parser.add_argument(

1264 - "--tokenizer-mode",

1265 - type=str,

1266 - default="auto",

1267 - choices=["auto", "slow", "mistral", "custom"],

1268 - help='The tokenizer mode.\n\n* "auto" will use the '

1269 - 'fast tokenizer if available.\n* "slow" will '

1270 - "always use the slow tokenizer. \n* "

1271 - '"mistral" will always use the `mistral_common` tokenizer. \n*'

1272 - '"custom" will use --tokenizer to select the preregistered tokenizer.',

1273 - )

1274 -

1275 - parser.add_argument(

1276 - "--served-model-name",

1277 - type=str,

1278 - default=None,

1279 - help="The model name used in the API. "

1280 - "If not specified, the model name will be the "

1281 - "same as the ``--model`` argument. ",

1282 - )

1283 -

1284 - parser.add_argument(

1285 - "--lora-modules",

1286 - nargs="+",

1287 - default=None,

1288 - help="A subset of LoRA module names passed in when "

1289 - "launching the server. For each request, the "

1290 - "script chooses a LoRA module at random.",

1291 - )

1292 -

1293 - parser.add_argument(

1294 - "--ramp-up-strategy",

1295 - type=str,

1296 - default=None,

1297 - choices=["linear", "exponential"],

1298 - help="The ramp-up strategy. This would be used to "

1299 - "ramp up the request rate from initial RPS to final "

1300 - "RPS rate (specified by --ramp-up-start-rps and --ramp-up-end-rps). "

1301 - "over the duration of the benchmark.",

1302 - )

1303 - parser.add_argument(

1304 - "--ramp-up-start-rps",

1305 - type=int,

1306 - default=None,

1307 - help="The starting request rate for ramp-up (RPS). "

1308 - "Needs to be specified when --ramp-up-strategy is used.",

1309 - )

1310 - parser.add_argument(

1311 - "--ramp-up-end-rps",

1312 - type=int,

1313 - default=None,

1314 - help="The ending request rate for ramp-up (RPS). "

1315 - "Needs to be specified when --ramp-up-strategy is used.",

1316 - )

5 +if __name__ == "__main__":

6 + print("""DEPRECATED: This script has been moved to the vLLM CLI.

1318 - return parser

8 +Please use the following command instead:

9 + vllm bench serve

11 +For help with the new command, run:

12 + vllm bench serve --help

1321 -if __name__ == "__main__":

1322 - parser = create_argument_parser()

1323 - args = parser.parse_args()

1324 - main(args)

14 +Alternatively, you can run the new command directly with:

15 + python -m vllm.entrypoints.cli.main bench serve --help

16 +""")

17 + sys.exit(1)

benchmarks/benchmark_throughput.py

+11 -735

@@ -1,741 +1,17 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Benchmark offline inference throughput."""

3 +import sys

5 -import argparse

6 -import dataclasses

7 -import json

8 -import os

9 -import random

10 -import time

11 -import warnings

12 -from typing import Any, Optional, Union

13 -

14 -import torch

15 -import uvloop

16 -from tqdm import tqdm

17 -from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizerBase

18 -from typing_extensions import deprecated

19 -

20 -from benchmark_dataset import (

21 - AIMODataset,

22 - BurstGPTDataset,

23 - ConversationDataset,

24 - InstructCoderDataset,

25 - RandomDataset,

26 - SampleRequest,

27 - ShareGPTDataset,

28 - SonnetDataset,

29 - VisionArenaDataset,

30 -)

31 -from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json

32 -from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs

33 -from vllm.entrypoints.openai.api_server import (

34 - build_async_engine_client_from_engine_args,

35 -)

36 -from vllm.inputs import TextPrompt, TokensPrompt

37 -from vllm.lora.request import LoRARequest

38 -from vllm.outputs import RequestOutput

39 -from vllm.sampling_params import BeamSearchParams

40 -from vllm.utils import FlexibleArgumentParser, merge_async_iterators

41 -

42 -

43 -def run_vllm(

44 - requests: list[SampleRequest],

45 - n: int,

46 - engine_args: EngineArgs,

47 - disable_detokenize: bool = False,

48 -) -> tuple[float, Optional[list[RequestOutput]]]:

49 - from vllm import LLM, SamplingParams

50 -

51 - llm = LLM(**dataclasses.asdict(engine_args))

52 - assert all(

53 - llm.llm_engine.model_config.max_model_len

54 - >= (request.prompt_len + request.expected_output_len)

55 - for request in requests

56 - ), (

57 - "Please ensure that max_model_len is greater than the sum of"

58 - " prompt_len and expected_output_len for all requests."

59 - )

60 - # Add the requests to the engine.

61 - prompts: list[Union[TextPrompt, TokensPrompt]] = []

62 - sampling_params: list[SamplingParams] = []

63 - for request in requests:

64 - prompts.append(

65 - TokensPrompt(

66 - prompt_token_ids=request.prompt["prompt_token_ids"],

67 - multi_modal_data=request.multi_modal_data,

68 - )

69 - if "prompt_token_ids" in request.prompt

70 - else TextPrompt(

71 - prompt=request.prompt, multi_modal_data=request.multi_modal_data

72 - )

73 - )

74 - sampling_params.append(

75 - SamplingParams(

76 - n=n,

77 - temperature=1.0,

78 - top_p=1.0,

79 - ignore_eos=True,

80 - max_tokens=request.expected_output_len,

81 - detokenize=not disable_detokenize,

82 - )

83 - )

84 - lora_requests: Optional[list[LoRARequest]] = None

85 - if engine_args.enable_lora:

86 - lora_requests = [request.lora_request for request in requests]

87 -

88 - use_beam_search = False

89 -

90 - outputs = None

91 - if not use_beam_search:

92 - start = time.perf_counter()

93 - outputs = llm.generate(

94 - prompts, sampling_params, lora_request=lora_requests, use_tqdm=True

95 - )

96 - end = time.perf_counter()

97 - else:

98 - assert lora_requests is None, "BeamSearch API does not support LoRA"

99 - # output_len should be the same for all requests.

100 - output_len = requests[0].expected_output_len

101 - for request in requests:

102 - assert request.expected_output_len == output_len

103 - start = time.perf_counter()

104 - llm.beam_search(

105 - prompts,

106 - BeamSearchParams(

107 - beam_width=n,

108 - max_tokens=output_len,

109 - ignore_eos=True,

110 - ),

111 - )

112 - end = time.perf_counter()

113 - return end - start, outputs

114 -

115 -

116 -def run_vllm_chat(

117 - requests: list[SampleRequest],

118 - n: int,

119 - engine_args: EngineArgs,

120 - disable_detokenize: bool = False,

121 -) -> tuple[float, list[RequestOutput]]:

122 - """

123 - Run vLLM chat benchmark. This function is recommended ONLY for benchmarking

124 - multimodal models as it properly handles multimodal inputs and chat

125 - formatting. For non-multimodal models, use run_vllm() instead.

126 - """

127 - from vllm import LLM, SamplingParams

128 -

129 - llm = LLM(**dataclasses.asdict(engine_args))

130 -

131 - assert all(

132 - llm.llm_engine.model_config.max_model_len

133 - >= (request.prompt_len + request.expected_output_len)

134 - for request in requests

135 - ), (

136 - "Please ensure that max_model_len is greater than the sum of "

137 - "prompt_len and expected_output_len for all requests."

138 - )

139 -

140 - prompts = []

141 - sampling_params: list[SamplingParams] = []

142 - for request in requests:

143 - prompts.append(request.prompt)

144 - sampling_params.append(

145 - SamplingParams(

146 - n=n,

147 - temperature=1.0,

148 - top_p=1.0,

149 - ignore_eos=True,

150 - max_tokens=request.expected_output_len,

151 - detokenize=not disable_detokenize,

152 - )

153 - )

154 - start = time.perf_counter()

155 - outputs = llm.chat(prompts, sampling_params, use_tqdm=True)

156 - end = time.perf_counter()

157 - return end - start, outputs

158 -

159 -

160 -async def run_vllm_async(

161 - requests: list[SampleRequest],

162 - n: int,

163 - engine_args: AsyncEngineArgs,

164 - disable_frontend_multiprocessing: bool = False,

165 - disable_detokenize: bool = False,

166 -) -> float:

167 - from vllm import SamplingParams

168 -

169 - async with build_async_engine_client_from_engine_args(

170 - engine_args,

171 - disable_frontend_multiprocessing=disable_frontend_multiprocessing,

172 - ) as llm:

173 - model_config = await llm.get_model_config()

174 - assert all(

175 - model_config.max_model_len

176 - >= (request.prompt_len + request.expected_output_len)

177 - for request in requests

178 - ), (

179 - "Please ensure that max_model_len is greater than the sum of"

180 - " prompt_len and expected_output_len for all requests."

181 - )

182 -

183 - # Add the requests to the engine.

184 - prompts: list[Union[TextPrompt, TokensPrompt]] = []

185 - sampling_params: list[SamplingParams] = []

186 - lora_requests: list[Optional[LoRARequest]] = []

187 - for request in requests:

188 - prompts.append(

189 - TokensPrompt(

190 - prompt_token_ids=request.prompt["prompt_token_ids"],

191 - multi_modal_data=request.multi_modal_data,

192 - )

193 - if "prompt_token_ids" in request.prompt

194 - else TextPrompt(

195 - prompt=request.prompt, multi_modal_data=request.multi_modal_data

196 - )

197 - )

198 - sampling_params.append(

199 - SamplingParams(

200 - n=n,

201 - temperature=1.0,

202 - top_p=1.0,

203 - ignore_eos=True,

204 - max_tokens=request.expected_output_len,

205 - detokenize=not disable_detokenize,

206 - )

207 - )

208 - lora_requests.append(request.lora_request)

209 -

210 - generators = []

211 - start = time.perf_counter()

212 - for i, (prompt, sp, lr) in enumerate(

213 - zip(prompts, sampling_params, lora_requests)

214 - ):

215 - generator = llm.generate(prompt, sp, lora_request=lr, request_id=f"test{i}")

216 - generators.append(generator)

217 - all_gens = merge_async_iterators(*generators)

218 - async for i, res in all_gens:

219 - pass

220 - end = time.perf_counter()

221 - return end - start

222 -

223 -

224 -def run_hf(

225 - requests: list[SampleRequest],

226 - model: str,

227 - tokenizer: PreTrainedTokenizerBase,

228 - n: int,

229 - max_batch_size: int,

230 - trust_remote_code: bool,

231 - disable_detokenize: bool = False,

232 -) -> float:

233 - llm = AutoModelForCausalLM.from_pretrained(

234 - model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code

235 - )

236 - if llm.config.model_type == "llama":

237 - # To enable padding in the HF backend.

238 - tokenizer.pad_token = tokenizer.eos_token

239 - llm = llm.cuda()

240 -

241 - pbar = tqdm(total=len(requests))

242 - start = time.perf_counter()

243 - batch: list[str] = []

244 - max_prompt_len = 0

245 - max_output_len = 0

246 - for i in range(len(requests)):

247 - prompt = requests[i].prompt

248 - prompt_len = requests[i].prompt_len

249 - output_len = requests[i].expected_output_len

250 - # Add the prompt to the batch.

251 - batch.append(prompt)

252 - max_prompt_len = max(max_prompt_len, prompt_len)

253 - max_output_len = max(max_output_len, output_len)

254 - if len(batch) < max_batch_size and i != len(requests) - 1:

255 - # Check if we can add more requests to the batch.

256 - next_prompt_len = requests[i + 1].prompt_len

257 - next_output_len = requests[i + 1].expected_output_len

258 - if (

259 - max(max_prompt_len, next_prompt_len)

260 - + max(max_output_len, next_output_len)

261 - ) <= 2048:

262 - # We can add more requests to the batch.

263 - continue

264 -

265 - # Generate the sequences.

266 - input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids

267 - llm_outputs = llm.generate(

268 - input_ids=input_ids.cuda(),

269 - do_sample=True,

270 - num_return_sequences=n,

271 - temperature=1.0,

272 - top_p=1.0,

273 - use_cache=True,

274 - max_new_tokens=max_output_len,

275 - )

276 - if not disable_detokenize:

277 - # Include the decoding time.

278 - tokenizer.batch_decode(llm_outputs, skip_special_tokens=True)

279 - pbar.update(len(batch))

280 -

281 - # Clear the batch.

282 - batch = []

283 - max_prompt_len = 0

284 - max_output_len = 0

285 - end = time.perf_counter()

286 - return end - start

287 -

288 -

289 -def run_mii(

290 - requests: list[SampleRequest],

291 - model: str,

292 - tensor_parallel_size: int,

293 - output_len: int,

294 -) -> float:

295 - from mii import client, serve

296 -

297 - llm = serve(model, tensor_parallel=tensor_parallel_size)

298 - prompts = [request.prompt for request in requests]

299 -

300 - start = time.perf_counter()

301 - llm.generate(prompts, max_new_tokens=output_len)

302 - end = time.perf_counter()

303 - client = client(model)

304 - client.terminate_server()

305 - return end - start

306 -

307 -

308 -def save_to_pytorch_benchmark_format(

309 - args: argparse.Namespace, results: dict[str, Any]

310 -) -> None:

311 - pt_records = convert_to_pytorch_benchmark_format(

312 - args=args,

313 - metrics={

314 - "requests_per_second": [results["requests_per_second"]],

315 - "tokens_per_second": [results["tokens_per_second"]],

316 - },

317 - extra_info={

318 - k: results[k] for k in ["elapsed_time", "num_requests", "total_num_tokens"]

319 - },

320 - )

321 - if pt_records:

322 - # Don't use json suffix here as we don't want CI to pick it up

323 - pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"

324 - write_to_json(pt_file, pt_records)

325 -

326 -

327 -def get_requests(args, tokenizer):

328 - # Common parameters for all dataset types.

329 - common_kwargs = {

330 - "dataset_path": args.dataset_path,

331 - "random_seed": args.seed,

332 - }

333 - sample_kwargs = {

334 - "tokenizer": tokenizer,

335 - "lora_path": args.lora_path,

336 - "max_loras": args.max_loras,

337 - "num_requests": args.num_prompts,

338 - "input_len": args.input_len,

339 - "output_len": args.output_len,

340 - }

341 -

342 - if args.dataset_path is None or args.dataset_name == "random":

343 - sample_kwargs["range_ratio"] = args.random_range_ratio

344 - sample_kwargs["prefix_len"] = args.prefix_len

345 - dataset_cls = RandomDataset

346 - elif args.dataset_name == "sharegpt":

347 - dataset_cls = ShareGPTDataset

348 - if args.backend == "vllm-chat":

349 - sample_kwargs["enable_multimodal_chat"] = True

350 - elif args.dataset_name == "sonnet":

351 - assert tokenizer.chat_template or tokenizer.default_chat_template, (

352 - "Tokenizer/model must have chat template for sonnet dataset."

353 - )

354 - dataset_cls = SonnetDataset

355 - sample_kwargs["prefix_len"] = args.prefix_len

356 - sample_kwargs["return_prompt_formatted"] = True

357 - elif args.dataset_name == "burstgpt":

358 - dataset_cls = BurstGPTDataset

359 - elif args.dataset_name == "hf":

360 - common_kwargs["no_stream"] = args.no_stream

361 - if args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS:

362 - dataset_cls = VisionArenaDataset

363 - common_kwargs["dataset_subset"] = None

364 - common_kwargs["dataset_split"] = "train"

365 - sample_kwargs["enable_multimodal_chat"] = True

366 - elif args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS:

367 - dataset_cls = InstructCoderDataset

368 - common_kwargs["dataset_split"] = "train"

369 - elif args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS:

370 - dataset_cls = ConversationDataset

371 - common_kwargs["dataset_subset"] = args.hf_subset

372 - common_kwargs["dataset_split"] = args.hf_split

373 - sample_kwargs["enable_multimodal_chat"] = True

374 - elif args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS:

375 - dataset_cls = AIMODataset

376 - common_kwargs["dataset_subset"] = None

377 - common_kwargs["dataset_split"] = "train"

378 - else:

379 - raise ValueError(f"Unknown dataset name: {args.dataset_name}")

380 - # Remove None values

381 - sample_kwargs = {k: v for k, v in sample_kwargs.items() if v is not None}

382 - return dataset_cls(**common_kwargs).sample(**sample_kwargs)

383 -

384 -

385 -@deprecated(

386 - "benchmark_throughput.py is deprecated and will be removed in a "

387 - "future version. Please use 'vllm bench throughput' instead.",

388 -)

389 -def main(args: argparse.Namespace):

390 - if args.seed is None:

391 - args.seed = 0

392 - print(args)

393 - random.seed(args.seed)

394 - # Sample the requests.

395 - tokenizer = AutoTokenizer.from_pretrained(

396 - args.tokenizer, trust_remote_code=args.trust_remote_code

397 - )

398 - requests = get_requests(args, tokenizer)

399 - is_multi_modal = any(request.multi_modal_data is not None for request in requests)

400 - request_outputs: Optional[list[RequestOutput]] = None

401 - if args.backend == "vllm":

402 - if args.async_engine:

403 - elapsed_time = uvloop.run(

404 - run_vllm_async(

405 - requests,

406 - args.n,

407 - AsyncEngineArgs.from_cli_args(args),

408 - args.disable_frontend_multiprocessing,

409 - args.disable_detokenize,

410 - )

411 - )

412 - else:

413 - elapsed_time, request_outputs = run_vllm(

414 - requests,

415 - args.n,

416 - EngineArgs.from_cli_args(args),

417 - args.disable_detokenize,

418 - )

419 - elif args.backend == "hf":

420 - assert args.tensor_parallel_size == 1

421 - elapsed_time = run_hf(

422 - requests,

423 - args.model,

424 - tokenizer,

425 - args.n,

426 - args.hf_max_batch_size,

427 - args.trust_remote_code,

428 - args.disable_detokenize,

429 - )

430 - elif args.backend == "mii":

431 - elapsed_time = run_mii(

432 - requests, args.model, args.tensor_parallel_size, args.output_len

433 - )

434 - elif args.backend == "vllm-chat":

435 - elapsed_time, request_outputs = run_vllm_chat(

436 - requests, args.n, EngineArgs.from_cli_args(args), args.disable_detokenize

437 - )

438 - else:

439 - raise ValueError(f"Unknown backend: {args.backend}")

440 -

441 - if request_outputs:

442 - # Note: with the vllm and vllm-chat backends,

443 - # we have request_outputs, which we use to count tokens.

444 - total_prompt_tokens = 0

445 - total_output_tokens = 0

446 - for ro in request_outputs:

447 - if not isinstance(ro, RequestOutput):

448 - continue

449 - total_prompt_tokens += (

450 - len(ro.prompt_token_ids) if ro.prompt_token_ids else 0

451 - )

452 - total_output_tokens += sum(len(o.token_ids) for o in ro.outputs if o)

453 - total_num_tokens = total_prompt_tokens + total_output_tokens

454 - else:

455 - total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)

456 - total_output_tokens = sum(r.expected_output_len for r in requests)

457 - total_prompt_tokens = total_num_tokens - total_output_tokens

458 -

459 - if is_multi_modal and args.backend != "vllm-chat":

460 - print(

461 - "\033[91mWARNING\033[0m: Multi-modal request with "

462 - f"{args.backend} backend detected. The "

463 - "following metrics are not accurate because image tokens are not"

464 - " counted. See vllm-project/vllm/issues/9778 for details."

465 - )

466 - # TODO(vllm-project/vllm/issues/9778): Count multi-modal token length.

467 - # vllm-chat backend counts the image tokens now

468 -

469 - print(

470 - f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "

471 - f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "

472 - f"{total_output_tokens / elapsed_time:.2f} output tokens/s"

473 - )

474 - print(f"Total num prompt tokens: {total_prompt_tokens}")

475 - print(f"Total num output tokens: {total_output_tokens}")

476 -

477 - # Output JSON results if specified

478 - if args.output_json:

479 - results = {

480 - "elapsed_time": elapsed_time,

481 - "num_requests": len(requests),

482 - "total_num_tokens": total_num_tokens,

483 - "requests_per_second": len(requests) / elapsed_time,

484 - "tokens_per_second": total_num_tokens / elapsed_time,

485 - }

486 - with open(args.output_json, "w") as f:

487 - json.dump(results, f, indent=4)

488 - save_to_pytorch_benchmark_format(args, results)

489 -

490 -

491 -def validate_args(args):

492 - """

493 - Validate command-line arguments.

494 - """

495 -

496 - # === Deprecation and Defaulting ===

497 - if args.dataset is not None:

498 - warnings.warn(

499 - "The '--dataset' argument will be deprecated in the next release. "

500 - "Please use '--dataset-name' and '--dataset-path' instead.",

501 - stacklevel=2,

502 - )

503 - args.dataset_path = args.dataset

504 -

505 - if not getattr(args, "tokenizer", None):

506 - args.tokenizer = args.model

507 -

508 - # === Backend Validation ===

509 - valid_backends = {"vllm", "hf", "mii", "vllm-chat"}

510 - if args.backend not in valid_backends:

511 - raise ValueError(f"Unsupported backend: {args.backend}")

512 -

513 - # === Dataset Configuration ===

514 - if not args.dataset and not args.dataset_path:

515 - print("When dataset path is not set, it will default to random dataset")

516 - args.dataset_name = "random"

517 - if args.input_len is None:

518 - raise ValueError("input_len must be provided for a random dataset")

519 -

520 - # === Dataset Name Specific Checks ===

521 - # --hf-subset and --hf-split: only used

522 - # when dataset_name is 'hf'

523 - if args.dataset_name != "hf" and (

524 - getattr(args, "hf_subset", None) is not None

525 - or getattr(args, "hf_split", None) is not None

526 - ):

527 - warnings.warn(

528 - "--hf-subset and --hf-split will be ignored \

529 - since --dataset-name is not 'hf'.",

530 - stacklevel=2,

531 - )

532 - elif args.dataset_name == "hf":

533 - if args.dataset_path in (

534 - VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()

535 - | ConversationDataset.SUPPORTED_DATASET_PATHS

536 - ):

537 - assert args.backend == "vllm-chat", (

538 - f"{args.dataset_path} needs to use vllm-chat as the backend."

539 - ) # noqa: E501

540 - elif args.dataset_path in (

541 - InstructCoderDataset.SUPPORTED_DATASET_PATHS

542 - | AIMODataset.SUPPORTED_DATASET_PATHS

543 - ):

544 - assert args.backend == "vllm", (

545 - f"{args.dataset_path} needs to use vllm as the backend."

546 - ) # noqa: E501

547 - else:

548 - raise ValueError(f"{args.dataset_path} is not supported by hf dataset.")

549 -

550 - # --random-range-ratio: only used when dataset_name is 'random'

551 - if args.dataset_name != "random" and args.random_range_ratio is not None:

552 - warnings.warn(

553 - "--random-range-ratio will be ignored since \

554 - --dataset-name is not 'random'.",

555 - stacklevel=2,

556 - )

557 -

558 - # --prefix-len: only used when dataset_name is 'random', 'sonnet', or not

559 - # set.

560 - if (

561 - args.dataset_name not in {"random", "sonnet", None}

562 - and args.prefix_len is not None

563 - ):

564 - warnings.warn(

565 - "--prefix-len will be ignored since --dataset-name\

566 - is not 'random', 'sonnet', or not set.",

567 - stacklevel=2,

568 - )

569 -

570 - # === LoRA Settings ===

571 - if getattr(args, "enable_lora", False) and args.backend != "vllm":

572 - raise ValueError("LoRA benchmarking is only supported for vLLM backend")

573 - if getattr(args, "enable_lora", False) and args.lora_path is None:

574 - raise ValueError("LoRA path must be provided when enable_lora is True")

575 -

576 - # === Backend-specific Validations ===

577 - if args.backend == "hf" and args.hf_max_batch_size is None:

578 - raise ValueError("HF max batch size is required for HF backend")

579 - if args.backend != "hf" and args.hf_max_batch_size is not None:

580 - raise ValueError("HF max batch size is only for HF backend.")

581 -

582 - if (

583 - args.backend in {"hf", "mii"}

584 - and getattr(args, "quantization", None) is not None

585 - ):

586 - raise ValueError("Quantization is only for vLLM backend.")

587 -

588 - if args.backend == "mii" and args.dtype != "auto":

589 - raise ValueError("dtype must be auto for MII backend.")

590 - if args.backend == "mii" and args.n != 1:

591 - raise ValueError("n must be 1 for MII backend.")

592 - if args.backend == "mii" and args.tokenizer != args.model:

593 - raise ValueError("Tokenizer must be the same as the model for MII backend.")

594 -

595 - # --data-parallel is not supported currently.

596 - # https://github.com/vllm-project/vllm/issues/16222

597 - if args.data_parallel_size > 1:

598 - raise ValueError(

599 - "Data parallel is not supported in offline benchmark, "

600 - "please use benchmark serving instead"

601 - )

602 -

603 -

604 -def create_argument_parser():

605 - parser = FlexibleArgumentParser(description="Benchmark the throughput.")

606 - parser.add_argument(

607 - "--backend",

608 - type=str,

609 - choices=["vllm", "hf", "mii", "vllm-chat"],

610 - default="vllm",

611 - )

612 - parser.add_argument(

613 - "--dataset-name",

614 - type=str,

615 - choices=["sharegpt", "random", "sonnet", "burstgpt", "hf"],

616 - help="Name of the dataset to benchmark on.",

617 - default="sharegpt",

618 - )

619 - parser.add_argument(

620 - "--no-stream",

621 - action="store_true",

622 - help="Do not load the dataset in streaming mode.",

623 - )

624 - parser.add_argument(

625 - "--dataset",

626 - type=str,

627 - default=None,

628 - help="Path to the ShareGPT dataset, will be deprecated in\

629 - the next release. The dataset is expected to "

630 - "be a json in form of list[dict[..., conversations: "

631 - "list[dict[..., value: <prompt_or_response>]]]]",

632 - )

633 - parser.add_argument(

634 - "--dataset-path", type=str, default=None, help="Path to the dataset"

635 - )

636 - parser.add_argument(

637 - "--input-len",

638 - type=int,

639 - default=None,

640 - help="Input prompt length for each request",

641 - )

642 - parser.add_argument(

643 - "--output-len",

644 - type=int,

645 - default=None,

646 - help="Output length for each request. Overrides the "

647 - "output length from the dataset.",

648 - )

649 - parser.add_argument(

650 - "--n", type=int, default=1, help="Number of generated sequences per prompt."

651 - )

652 - parser.add_argument(

653 - "--num-prompts", type=int, default=1000, help="Number of prompts to process."

654 - )

655 - parser.add_argument(

656 - "--hf-max-batch-size",

657 - type=int,

658 - default=None,

659 - help="Maximum batch size for HF backend.",

660 - )

661 - parser.add_argument(

662 - "--output-json",

663 - type=str,

664 - default=None,

665 - help="Path to save the throughput results in JSON format.",

666 - )

667 - parser.add_argument(

668 - "--async-engine",

669 - action="store_true",

670 - default=False,

671 - help="Use vLLM async engine rather than LLM class.",

672 - )

673 - parser.add_argument(

674 - "--disable-frontend-multiprocessing",

675 - action="store_true",

676 - default=False,

677 - help="Disable decoupled async engine frontend.",

678 - )

679 - parser.add_argument(

680 - "--disable-detokenize",

681 - action="store_true",

682 - help=(

683 - "Do not detokenize the response (i.e. do not include "

684 - "detokenization time in the measurement)"

685 - ),

686 - )

687 - # LoRA

688 - parser.add_argument(

689 - "--lora-path",

690 - type=str,

691 - default=None,

692 - help="Path to the LoRA adapters to use. This can be an absolute path, "

693 - "a relative path, or a Hugging Face model identifier.",

694 - )

695 - parser.add_argument(

696 - "--prefix-len",

697 - type=int,

698 - default=None,

699 - help=f"Number of prefix tokens to be used in RandomDataset "

700 - "and SonnetDataset. For RandomDataset, the total input "

701 - "length is the sum of prefix-len (default: "

702 - f"{RandomDataset.DEFAULT_PREFIX_LEN}) and a random context length "

703 - "sampled from [input_len * (1 - range_ratio), "

704 - "input_len * (1 + range_ratio)]. For SonnetDataset, "

705 - f"prefix_len (default: {SonnetDataset.DEFAULT_PREFIX_LEN}) "

706 - "controls how much of the input is fixed lines versus "

707 - "random lines, but the total input length remains approximately "

708 - "input_len tokens.",

709 - )

710 - # random dataset

711 - parser.add_argument(

712 - "--random-range-ratio",

713 - type=float,

714 - default=None,

715 - help=f"Range ratio (default : {RandomDataset.DEFAULT_RANGE_RATIO}) "

716 - "for sampling input/output length, "

717 - "used only for RandomDataset. Must be in the range [0, 1) to "

718 - "define a symmetric sampling range "

719 - "[length * (1 - range_ratio), length * (1 + range_ratio)].",

720 - )

721 -

722 - # hf dataset

723 - parser.add_argument(

724 - "--hf-subset", type=str, default=None, help="Subset of the HF dataset."

725 - )

726 - parser.add_argument(

727 - "--hf-split", type=str, default=None, help="Split of the HF dataset."

728 - )

729 -

730 - parser = AsyncEngineArgs.add_cli_args(parser)

5 +if __name__ == "__main__":

6 + print("""DEPRECATED: This script has been moved to the vLLM CLI.

732 - return parser

8 +Please use the following command instead:

9 + vllm bench throughput

11 +For help with the new command, run:

12 + vllm bench throughput --help

735 -if __name__ == "__main__":

736 - parser = create_argument_parser()

737 - args = parser.parse_args()

738 - if args.tokenizer is None:

739 - args.tokenizer = args.model

740 - validate_args(args)

741 - main(args)

14 +Alternatively, you can run the new command directly with:

15 + python -m vllm.entrypoints.cli.main bench throughput --help

16 +""")

17 + sys.exit(1)