@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 58abe35455

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 19:45 UTC

[Benchmarks] Make detokenization optional in benchmark scripts (#11697)

Signed-off-by: Jeremy Arnold <Jeremy.Arnold@amd.com>

Jeremy Arnold committed Mar 7, 2025 at 10:09 UTC 58abe35455685a0dd6f1484614c663eff858aa44

4 files changed +45 -7

benchmarks/benchmark_latency.py

@@ -52,6 +52,7 @@ def main(args: argparse.Namespace):

52 top_p=1.0,

53 ignore_eos=True,

54 max_tokens=args.output_len,

55 + detokenize=not args.disable_detokenize,

56 )

57 print(sampling_params)

58 dummy_prompt_token_ids = np.random.randint(10000,

@@ -173,6 +174,12 @@ if __name__ == "__main__":

174 default=None,

175 help="Path to save the latency results in JSON format.",

176 )

177 + parser.add_argument(

178 + "--disable-detokenize",

179 + action="store_true",

180 + help=("Do not detokenize responses (i.e. do not include "

181 + "detokenization time in the latency measurement)"),

182 + )

183

184 parser = EngineArgs.add_cli_args(parser)

185 args = parser.parse_args()

benchmarks/benchmark_prefix_caching.py

+9 -1

@@ -194,7 +194,9 @@ def main(args):

194

195 llm = LLM(**dataclasses.asdict(engine_args))

196

197 - sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)

197 + sampling_params = SamplingParams(temperature=0,

198 + max_tokens=args.output_len,

199 + detokenize=not args.disable_detokenize)

200

201 print("Testing filtered requests")

202 prompts = repeat_and_sort_requests(filtered_requests,

@@ -243,6 +245,12 @@ if __name__ == "__main__":

245 "subtract this length when filtering prompts. Only used "

246 "when dataset-path is not provided.",

247 )

248 + parser.add_argument(

249 + '--disable-detokenize',

250 + action='store_true',

251 + help=("Do not detokenize responses (i.e. do not include "

252 + "detokenization time in the latency measurement)"),

253 + )

254

255 parser = EngineArgs.add_cli_args(parser)

256 args = parser.parse_args()

benchmarks/benchmark_prioritization.py

+11 -2

@@ -23,7 +23,7 @@ def sample_requests(

23 num_requests: int,

24 tokenizer: PreTrainedTokenizerBase,

25 fixed_output_len: Optional[int],

26 -) -> list[tuple[str, int, int]]:

26 +) -> list[tuple[str, int, int, int]]:

27 if fixed_output_len is not None and fixed_output_len < 4:

28 raise ValueError("output_len too small")

@@ -71,6 +71,7 @@ def run_vllm(

71 requests: list[tuple[str, int, int]],

72 n: int,

73 engine_args: EngineArgs,

74 + disable_detokenize: bool = False,

75 ) -> float:

76 from vllm import LLM, SamplingParams

77 llm = LLM(**dataclasses.asdict(engine_args))

@@ -95,6 +96,7 @@ def run_vllm(

96 top_p=1.0,

97 ignore_eos=True,

98 max_tokens=output_len,

99 + detokenize=not disable_detokenize,

100 ))

101

102 start = time.perf_counter()

@@ -121,7 +123,8 @@ def main(args: argparse.Namespace):

123

124 if args.backend == "vllm":

125 elapsed_time = run_vllm(requests, args.n,

124 - EngineArgs.from_cli_args(args))

126 + EngineArgs.from_cli_args(args),

127 + args.disable_detokenize)

128 else:

129 raise ValueError(f"Unknown backend: {args.backend}")

130 total_num_tokens = sum(prompt_len + output_len

@@ -174,6 +177,12 @@ if __name__ == "__main__":

177 type=str,

178 default=None,

179 help='Path to save the throughput results in JSON format.')

180 + parser.add_argument(

181 + '--disable-detokenize',

182 + action='store_true',

183 + help=("Do not detokenize responses (i.e. do not include "

184 + "detokenization time in the latency measurement)"),

185 + )

186

187 parser = EngineArgs.add_cli_args(parser)

188 args = parser.parse_args()

benchmarks/benchmark_throughput.py

+18 -4

@@ -168,6 +168,7 @@ def run_vllm(

168 requests: list[SampleRequest],

169 n: int,

170 engine_args: EngineArgs,

171 + disable_detokenize: bool = False,

172 ) -> float:

173 from vllm import LLM, SamplingParams

174 llm = LLM(**dataclasses.asdict(engine_args))

@@ -194,6 +195,7 @@ def run_vllm(

195 top_p=1.0,

196 ignore_eos=True,

197 max_tokens=request.expected_output_len,

198 + detokenize=not disable_detokenize,

199 ))

200 lora_requests: Optional[list[LoRARequest]] = None

201 if engine_args.enable_lora:

@@ -232,6 +234,7 @@ async def run_vllm_async(

234 n: int,

235 engine_args: AsyncEngineArgs,

236 disable_frontend_multiprocessing: bool = False,

237 + disable_detokenize: bool = False,

238 ) -> float:

239 from vllm import SamplingParams

240

@@ -262,6 +265,7 @@ async def run_vllm_async(

265 top_p=1.0,

266 ignore_eos=True,

267 max_tokens=request.expected_output_len,

268 + detokenize=not disable_detokenize,

269 ))

270 lora_requests.append(request.lora_request)

271

@@ -288,6 +292,7 @@ def run_hf(

292 n: int,

293 max_batch_size: int,

294 trust_remote_code: bool,

295 + disable_detokenize: bool = False,

296 ) -> float:

297 llm = AutoModelForCausalLM.from_pretrained(

298 model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)

@@ -327,8 +332,9 @@ def run_hf(

332 use_cache=True,

333 max_new_tokens=max_output_len,

334 )

330 - # Include the decoding time.

331 - tokenizer.batch_decode(llm_outputs, skip_special_tokens=True)

335 + if not disable_detokenize:

336 + # Include the decoding time.

337 + tokenizer.batch_decode(llm_outputs, skip_special_tokens=True)

338 pbar.update(len(batch))

339

340 # Clear the batch.

@@ -440,14 +446,17 @@ def main(args: argparse.Namespace):

446 args.n,

447 AsyncEngineArgs.from_cli_args(args),

448 args.disable_frontend_multiprocessing,

449 + args.disable_detokenize,

450 ))

451 else:

452 elapsed_time = run_vllm(requests, args.n,

446 - EngineArgs.from_cli_args(args))

453 + EngineArgs.from_cli_args(args),

454 + args.disable_detokenize)

455 elif args.backend == "hf":

456 assert args.tensor_parallel_size == 1

457 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,

450 - args.hf_max_batch_size, args.trust_remote_code)

458 + args.hf_max_batch_size, args.trust_remote_code,

459 + args.disable_detokenize)

460 elif args.backend == "mii":

461 elapsed_time = run_mii(requests, args.model, args.tensor_parallel_size,

462 args.output_len)

@@ -526,6 +535,11 @@ if __name__ == "__main__":

535 action='store_true',

536 default=False,

537 help="Disable decoupled async engine frontend.")

538 + parser.add_argument(

539 + "--disable-detokenize",

540 + action="store_true",

541 + help=("Do not detokenize the response (i.e. do not include "

542 + "detokenization time in the measurement)"))

543 # LoRA

544 parser.add_argument(

545 "--lora-path",