@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ecbb14b81

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Benchmarks] add benchmark for embedding models (#23000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>

Jiangyun Zhu committed Aug 26, 2025 at 14:57 UTC 3ecbb14b814f9559bce88fa62ea8b5deedbc6076

3 files changed +274 -107

vllm/benchmarks/datasets.py

+45 -22

@@ -73,7 +73,7 @@ class SampleRequest:

73 Represents a single inference request for benchmarking.

74 """

76 - prompt: Union[str, Any]

76 + prompt: Union[str, list[str]]

77 prompt_len: int

78 expected_output_len: int

79 multi_modal_data: Optional[

@@ -409,6 +409,7 @@ class RandomDataset(BenchmarkDataset):

409 range_ratio: float = DEFAULT_RANGE_RATIO,

410 input_len: int = DEFAULT_INPUT_LEN,

411 output_len: int = DEFAULT_OUTPUT_LEN,

412 + batchsize: int = 1,

413 **kwargs,

414 ) -> list[SampleRequest]:

415

@@ -439,6 +440,21 @@ class RandomDataset(BenchmarkDataset):

440 request_id=request_id_prefix + str(i),

441 )

442 )

443 + # only used for embeddings benchmark.

444 + if batchsize > 1:

445 + batch_requests = []

446 + # Create batched requests

447 + for i in range(0, num_requests, batchsize):

448 + batch = requests[i : i + batchsize]

449 + batch_requests.append(

450 + SampleRequest(

451 + prompt=[req.prompt for req in batch],

452 + prompt_len=sum(req.prompt_len for req in batch),

453 + expected_output_len=0,

454 + request_id=request_id_prefix + str(i // batchsize),

455 + )

456 + )

457 + requests = batch_requests

458 return requests

459

460 def get_prefix(

@@ -475,8 +491,8 @@ class RandomDataset(BenchmarkDataset):

491 input_high = math.ceil(real_input_len * (1 + range_ratio))

492 output_low = math.floor(output_len * (1 - range_ratio))

493 output_high = math.ceil(output_len * (1 + range_ratio))

478 - # Ensure the lower bound for output length is at least 1 to

479 - # prevent sampling 0 tokens.

494 + # Ensure the lower bound for output length is at least 1 to

495 + # prevent sampling 0 tokens.

496 output_low = max(output_low, 1)

497

498 if input_low > input_high:

@@ -506,7 +522,6 @@ class RandomDataset(BenchmarkDataset):

522 size=num_requests)

523 return input_lens, output_lens, offsets

524

509 -

525 def generate_token_sequence(

526 self,

527 *,

@@ -1105,6 +1120,13 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1120 "context length sampled from [input_len * (1 - range_ratio), "

1121 "input_len * (1 + range_ratio)]."),

1122 )

1123 + random_group.add_argument(

1124 + "--random-batch-size",

1125 + type=int,

1126 + default=1,

1127 + help=("Batch size for random sampling. "

1128 + "Only used for embeddings benchmark."),

1129 + )

1130

1131 # random multimodal dataset options

1132 random_mm_group = parser.add_argument_group(

@@ -1196,8 +1218,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1218 ),

1219 )

1220

1199 -

1200 -

1221 hf_group = parser.add_argument_group("hf dataset options")

1222 hf_group.add_argument("--hf-subset",

1223 type=str,

@@ -1348,22 +1368,24 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1368 else:

1369 # For datasets that follow a similar structure, use a mapping.

1370 dataset_mapping = {

1351 - "sharegpt":

1352 - lambda: ShareGPTDataset(random_seed=args.seed,

1353 - dataset_path=args.dataset_path).sample(

1354 - tokenizer=tokenizer,

1355 - num_requests=args.num_prompts,

1356 - output_len=args.sharegpt_output_len,

1357 - request_id_prefix=args.request_id_prefix,

1358 - ),

1359 - "burstgpt":

1360 - lambda: BurstGPTDataset(random_seed=args.seed,

1361 - dataset_path=args.dataset_path).

1362 - sample(tokenizer=tokenizer, num_requests=args.num_prompts,

1363 - request_id_prefix=args.request_id_prefix,),

1364 - "random":

1365 - lambda: RandomDataset(random_seed=args.seed,

1366 - dataset_path=args.dataset_path).sample(

1371 + "sharegpt": lambda: ShareGPTDataset(

1372 + random_seed=args.seed, dataset_path=args.dataset_path

1373 + ).sample(

1374 + tokenizer=tokenizer,

1375 + num_requests=args.num_prompts,

1376 + output_len=args.sharegpt_output_len,

1377 + request_id_prefix=args.request_id_prefix,

1378 + ),

1379 + "burstgpt": lambda: BurstGPTDataset(

1380 + random_seed=args.seed, dataset_path=args.dataset_path

1381 + ).sample(

1382 + tokenizer=tokenizer,

1383 + num_requests=args.num_prompts,

1384 + request_id_prefix=args.request_id_prefix,

1385 + ),

1386 + "random": lambda: RandomDataset(

1387 + random_seed=args.seed, dataset_path=args.dataset_path

1388 + ).sample(

1389 tokenizer=tokenizer,

1390 num_requests=args.num_prompts,

1391 prefix_len=args.random_prefix_len,

@@ -1371,6 +1393,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1393 output_len=args.random_output_len,

1394 range_ratio=args.random_range_ratio,

1395 request_id_prefix=args.request_id_prefix,

1396 + batchsize=args.random_batch_size,

1397 ),

1398 "random-mm":

1399 lambda: RandomMultiModalDataset(

vllm/benchmarks/lib/endpoint_request_func.py

+53 -4

@@ -69,8 +69,8 @@ async def async_request_openai_completions(

69 ), "OpenAI Completions API URL must end with 'completions' or 'profile'."

71 payload = {

72 - "model": request_func_input.model_name \

73 - if request_func_input.model_name else request_func_input.model,

72 + "model": request_func_input.model_name

73 + if request_func_input.model_name else request_func_input.model,

74 "prompt": request_func_input.prompt,

75 "temperature": 0.0,

76 "repetition_penalty": 1.0,

@@ -135,7 +135,7 @@ async def async_request_openai_completions(

135 # Decoding phase

136 else:

137 output.itl.append(timestamp -

138 - most_recent_timestamp)

138 + most_recent_timestamp)

139

140 most_recent_timestamp = timestamp

141 generated_text += text or ""

@@ -254,7 +254,7 @@ async def async_request_openai_chat_completions(

254 # Decoding phase

255 else:

256 output.itl.append(timestamp -

257 - most_recent_timestamp)

257 + most_recent_timestamp)

258

259 generated_text += content or ""

260 elif usage := data.get("usage"):

@@ -394,12 +394,61 @@ async def async_request_openai_audio(

394 return output

395

396

397 +async def async_request_openai_embeddings(

398 + request_func_input: RequestFuncInput,

399 + session: aiohttp.ClientSession,

400 + pbar: Optional[tqdm] = None,

401 +):

402 + api_url = request_func_input.api_url

403 + assert api_url.endswith(

404 + "embeddings"

405 + ), "OpenAI Embeddings API URL must end with 'embeddings'."

406 +

407 + headers = {

408 + "Content-Type": "application/json",

409 + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",

410 + }

411 +

412 + payload = {

413 + "model": request_func_input.model,

414 + "input": request_func_input.prompt,

415 + }

416 +

417 + output = RequestFuncOutput()

418 + st = time.perf_counter()

419 + try:

420 + async with session.post(

421 + url=api_url,

422 + headers=headers,

423 + json=payload

424 + ) as response:

425 + if response.status == 200:

426 + output.latency = time.perf_counter() - st

427 + data = await response.json()

428 + output.success = True

429 + output.generated_text = ""

430 + output.prompt_len = data.get(

431 + "usage", {}).get(

432 + "prompt_tokens", 0)

433 + else:

434 + output.success = False

435 + output.error = response.reason or ""

436 + except Exception as e:

437 + output.success = False

438 + output.error = str(e)

439 +

440 + if pbar:

441 + pbar.update(1)

442 + return output

443 +

444 +

445 # TODO: Add more request functions for different API protocols.

446 ASYNC_REQUEST_FUNCS = {

447 "vllm": async_request_openai_completions,

448 "openai": async_request_openai_completions,

449 "openai-chat": async_request_openai_chat_completions,

450 "openai-audio": async_request_openai_audio,

451 + "openai-embeddings": async_request_openai_embeddings,

452 }

453

454 OPENAI_COMPATIBLE_BACKENDS = [

vllm/benchmarks/serve.py

+176 -81

@@ -4,7 +4,7 @@ r"""Benchmark online serving throughput.

5 On the server side, run one of the following commands

6 to launch the vLLM OpenAI API server:

7 - vllm serve <your_model> <engine arguments>

7 + vllm serve <your_model> <engine arguments>

9 On the client side, run:

10 vllm bench serve \

@@ -26,6 +26,7 @@ import warnings

26 from collections.abc import AsyncGenerator, Iterable

27 from dataclasses import dataclass

28 from datetime import datetime

29 +from enum import Enum

30 from typing import Any, Literal, Optional

32 import aiohttp

@@ -46,6 +47,11 @@ from vllm.transformers_utils.tokenizer import get_tokenizer

47 MILLISECONDS_TO_SECONDS_CONVERSION = 1000

50 +class TaskType(Enum):

51 + GENERATION = "generation"

52 + EMBEDDING = "embedding"

53 +

54 +

55 @dataclass

56 class BenchmarkMetrics:

57 completed: int

@@ -75,6 +81,16 @@ class BenchmarkMetrics:

81 std_e2el_ms: float

82 percentiles_e2el_ms: list[tuple[float, float]]

84 +@dataclass

85 +class EmbedBenchmarkMetrics:

86 + completed: int

87 + total_input: int

88 + request_throughput: float

89 + total_token_throughput :float

90 + mean_e2el_ms: float

91 + std_e2el_ms: float

92 + median_e2el_ms: float

93 + percentiles_e2el_ms: float

95 def _get_current_request_rate(

96 ramp_up_strategy: Optional[Literal["linear", "exponential"]],

@@ -146,11 +162,11 @@ async def get_request(

162 delay_ts = []

163 for request_index, request in enumerate(input_requests):

164 current_request_rate = _get_current_request_rate(ramp_up_strategy,

149 - ramp_up_start_rps,

150 - ramp_up_end_rps,

151 - request_index,

152 - total_requests,

153 - request_rate)

165 + ramp_up_start_rps,

166 + ramp_up_end_rps,

167 + request_index,

168 + total_requests,

169 + request_rate)

170 request_rates.append(current_request_rate)

171 if current_request_rate == float("inf"):

172 delay_ts.append(0)

@@ -160,7 +176,7 @@ async def get_request(

176 # Sample the request interval from the gamma distribution.

177 # If burstiness is 1, it follows exponential distribution.

178 delay_ts.append(np.random.gamma(shape=burstiness, scale=theta))

163 -

179 +

180 # Calculate the cumulative delay time from the first sent out requests.

181 for i in range(1, len(delay_ts)):

182 delay_ts[i] += delay_ts[i - 1]

@@ -170,11 +186,11 @@ async def get_request(

186 # logic would re-scale delay time to ensure the final delay_ts

187 # align with target_total_delay_s.

188 #

173 - # NOTE: If we simply accumulate the random delta values

174 - # from the gamma distribution, their sum would have 1-2% gap

189 + # NOTE: If we simply accumulate the random delta values

190 + # from the gamma distribution, their sum would have 1-2% gap

191 # from target_total_delay_s. The purpose of the following logic is to

176 - # close the gap for stablizing the throughput data

177 - # from different random seeds.

192 + # close the gap for stablizing the throughput data

193 + # from different random seeds.

194 target_total_delay_s = total_requests / request_rate

195 normalize_factor = target_total_delay_s / delay_ts[-1]

196 delay_ts = [delay * normalize_factor for delay in delay_ts]

@@ -189,6 +205,51 @@ async def get_request(

205 yield request, request_rates[request_index]

206

207

208 +def calculate_metrics_for_embeddings(

209 + outputs: list[RequestFuncOutput],

210 + dur_s: float,

211 + selected_percentiles: list[float]

212 +) -> EmbedBenchmarkMetrics:

213 + """Calculate the metrics for the embedding requests.

214 +

215 + Args:

216 + outputs: The outputs of the requests.

217 + dur_s: The duration of the benchmark.

218 + selected_percentiles: The percentiles to select.

219 +

220 + Returns:

221 + The calculated benchmark metrics.

222 + """

223 + total_input = 0

224 + completed = 0

225 + e2els: list[float] = []

226 + for i in range(len(outputs)):

227 + if outputs[i].success:

228 + e2els.append(outputs[i].latency)

229 + completed += 1

230 + total_input += outputs[i].prompt_len

231 +

232 + if completed == 0:

233 + warnings.warn(

234 + "All requests failed. This is likely due to a misconfiguration "

235 + "on the benchmark arguments.",

236 + stacklevel=2)

237 + metrics = EmbedBenchmarkMetrics(

238 + completed=completed,

239 + total_input=total_input,

240 + request_throughput=completed / dur_s,

241 + total_token_throughput=total_input / dur_s,

242 + mean_e2el_ms=np.mean(e2els or 0) * 1000,

243 + std_e2el_ms=np.std(e2els or 0) * 1000,

244 + median_e2el_ms=np.median(e2els or 0) * 1000,

245 + percentiles_e2el_ms=[

246 + (p, np.percentile(e2els or 0, p) * 1000)

247 + for p in selected_percentiles

248 + ],

249 + )

250 + return metrics

251 +

252 +

253 def calculate_metrics(

254 input_requests: list[SampleRequest],

255 outputs: list[RequestFuncOutput],

@@ -334,8 +395,16 @@ async def benchmark(

395 ramp_up_end_rps: Optional[int] = None,

396 ready_check_timeout_sec: int = 600,

397 ):

398 + task_type = (

399 + TaskType.EMBEDDING

400 + if api_url.endswith("/v1/embeddings")

401 + else TaskType.GENERATION

402 + )

403 if endpoint_type in ASYNC_REQUEST_FUNCS:

338 - request_func = ASYNC_REQUEST_FUNCS[endpoint_type]

404 + if task_type == TaskType.EMBEDDING:

405 + request_func = ASYNC_REQUEST_FUNCS["openai-embeddings"]

406 + else:

407 + request_func = ASYNC_REQUEST_FUNCS[endpoint_type]

408 else:

409 raise ValueError(f"Unknown endpoint_type: {endpoint_type}")

410

@@ -421,8 +490,8 @@ async def benchmark(

490 if profile_output.success:

491 print("Profiler started")

492

424 - distribution = ("Poisson process" if burstiness == 1.0

425 - else "Gamma distribution")

493 + distribution = ("Poisson process" if burstiness == 1.0

494 + else "Gamma distribution")

495

496 if ramp_up_strategy is not None:

497 print(f"Traffic ramp-up strategy: {ramp_up_strategy}.")

@@ -449,7 +518,7 @@ async def benchmark(

518 session=session,

519 pbar=pbar)

520 async with semaphore:

452 - return await request_func(request_func_input=request_func_input,

521 + return await request_func(request_func_input=request_func_input,

522 session=session,

523 pbar=pbar)

524

@@ -513,14 +582,22 @@ async def benchmark(

582

583 benchmark_duration = time.perf_counter() - benchmark_start_time

584

516 - metrics, actual_output_lens = calculate_metrics(

517 - input_requests=input_requests,

518 - outputs=outputs,

519 - dur_s=benchmark_duration,

520 - tokenizer=tokenizer,

521 - selected_percentiles=selected_percentiles,

522 - goodput_config_dict=goodput_config_dict,

523 - )

585 + if task_type == TaskType.GENERATION:

586 + metrics, actual_output_lens = calculate_metrics(

587 + input_requests=input_requests,

588 + outputs=outputs,

589 + dur_s=benchmark_duration,

590 + tokenizer=tokenizer,

591 + selected_percentiles=selected_percentiles,

592 + goodput_config_dict=goodput_config_dict,

593 + )

594 + else:

595 + metrics = calculate_metrics_for_embeddings(

596 + outputs=outputs,

597 + dur_s=benchmark_duration,

598 + selected_percentiles=selected_percentiles,

599 + )

600 + actual_output_lens = 0

601

602 print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))

603 print("{:<40} {:<10}".format("Successful requests:", metrics.completed))

@@ -529,39 +606,55 @@ async def benchmark(

606 max_concurrency))

607 if request_rate != float('inf'):

608 print("{:<40} {:<10.2f}".format("Request rate configured (RPS):",

532 - request_rate ))

609 + request_rate))

610 print("{:<40} {:<10.2f}".format("Benchmark duration (s):",

611 benchmark_duration))

612 print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))

536 - print("{:<40} {:<10}".format("Total generated tokens:",

537 - metrics.total_output))

613 + if isinstance(metrics, BenchmarkMetrics):

614 + print("{:<40} {:<10}".format(

615 + "Total generated tokens:", metrics.total_output))

616 print("{:<40} {:<10.2f}".format("Request throughput (req/s):",

617 metrics.request_throughput))

618 if goodput_config_dict:

619 print("{:<40} {:<10.2f}".format("Request goodput (req/s):",

620 metrics.request_goodput))

543 - print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",

544 - metrics.output_throughput))

621 + if isinstance(metrics, BenchmarkMetrics):

622 + print(

623 + "{:<40} {:<10.2f}".format(

624 + "Output token throughput (tok/s):", metrics.output_throughput

625 + )

626 + )

627 print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",

628 metrics.total_token_throughput))

629

548 - result = {

549 - "duration": benchmark_duration,

550 - "completed": metrics.completed,

551 - "total_input_tokens": metrics.total_input,

552 - "total_output_tokens": metrics.total_output,

553 - "request_throughput": metrics.request_throughput,

554 - "request_goodput":

555 - metrics.request_goodput if goodput_config_dict else None,

556 - "output_throughput": metrics.output_throughput,

557 - "total_token_throughput": metrics.total_token_throughput,

558 - "input_lens": [output.prompt_len for output in outputs],

559 - "output_lens": actual_output_lens,

560 - "ttfts": [output.ttft for output in outputs],

561 - "itls": [output.itl for output in outputs],

562 - "generated_texts": [output.generated_text for output in outputs],

563 - "errors": [output.error for output in outputs],

564 - }

630 + if isinstance(metrics, BenchmarkMetrics):

631 + result = {

632 + "duration": benchmark_duration,

633 + "completed": metrics.completed,

634 + "total_input_tokens": metrics.total_input,

635 + "total_output_tokens": metrics.total_output,

636 + "request_throughput": metrics.request_throughput,

637 + "request_goodput":

638 + metrics.request_goodput if goodput_config_dict else None,

639 + "output_throughput": metrics.output_throughput,

640 + "total_token_throughput": metrics.total_token_throughput,

641 + "input_lens": [output.prompt_len for output in outputs],

642 + "output_lens": actual_output_lens,

643 + "ttfts": [output.ttft for output in outputs],

644 + "itls": [output.itl for output in outputs],

645 + "generated_texts": [output.generated_text for output in outputs],

646 + "errors": [output.error for output in outputs],

647 + }

648 + else:

649 + result = {

650 + "duration": benchmark_duration,

651 + "completed": metrics.completed,

652 + "total_input_tokens": metrics.total_input,

653 + "request_throughput": metrics.request_throughput,

654 + "total_token_throughput": metrics.total_token_throughput,

655 + "input_lens": [output.prompt_len for output in outputs],

656 + "errors": [output.error for output in outputs],

657 + }

658

659 if rps_change_events:

660 result["rps_change_events"] = rps_change_events

@@ -598,10 +691,11 @@ async def benchmark(

691 value))

692 result[f"p{p_word}_{metric_attribute_name}_ms"] = value

693

601 - process_one_metric("ttft", "TTFT", "Time to First Token")

602 - process_one_metric("tpot", "TPOT",

603 - "Time per Output Token (excl. 1st token)")

604 - process_one_metric("itl", "ITL", "Inter-token Latency")

694 + if task_type == TaskType.GENERATION:

695 + process_one_metric("ttft", "TTFT", "Time to First Token")

696 + process_one_metric(

697 + "tpot", "TPOT", "Time per Output Token (excl. 1st token)")

698 + process_one_metric("itl", "ITL", "Inter-token Latency")

699 process_one_metric("e2el", "E2EL", "End-to-end Latency")

700

701 print("=" * 50)

@@ -732,7 +826,8 @@ def add_cli_args(parser: argparse.ArgumentParser):

826 "initiated, this argument will control how many are actually allowed "

827 "to execute at a time. This means that when used in combination, the "

828 "actual request rate may be lower than specified with --request-rate, "

735 - "if the server is not processing requests fast enough to keep up.")

829 + "if the server is not processing requests fast enough to keep up.",

830 + )

831

832 parser.add_argument(

833 "--model",

@@ -743,8 +838,7 @@ def add_cli_args(parser: argparse.ArgumentParser):

838 parser.add_argument(

839 "--tokenizer",

840 type=str,

746 - help=

747 - "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501

841 + help="Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501

842 )

843 parser.add_argument("--use-beam-search", action="store_true")

844 parser.add_argument(

@@ -968,6 +1062,7 @@ def add_cli_args(parser: argparse.ArgumentParser):

1062 def main(args: argparse.Namespace) -> dict[str, Any]:

1063 return asyncio.run(main_async(args))

1064

1065 +

1066 async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1067 print(args)

1068 random.seed(args.seed)

@@ -1046,32 +1141,32 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1141 gc.freeze()

1142

1143 benchmark_result = await benchmark(

1049 - endpoint_type=args.endpoint_type,

1050 - api_url=api_url,

1051 - base_url=base_url,

1052 - model_id=model_id,

1053 - model_name=model_name,

1054 - tokenizer=tokenizer,

1055 - input_requests=input_requests,

1056 - logprobs=args.logprobs,

1057 - request_rate=args.request_rate,

1058 - burstiness=args.burstiness,

1059 - disable_tqdm=args.disable_tqdm,

1060 - profile=args.profile,

1061 - selected_percentile_metrics=args.percentile_metrics.split(","),

1062 - selected_percentiles=[

1063 - float(p) for p in args.metric_percentiles.split(",")

1064 - ],

1065 - ignore_eos=args.ignore_eos,

1066 - goodput_config_dict=goodput_config_dict,

1067 - max_concurrency=args.max_concurrency,

1068 - lora_modules=args.lora_modules,

1069 - extra_body=sampling_params,

1070 - ramp_up_strategy=args.ramp_up_strategy,

1071 - ramp_up_start_rps=args.ramp_up_start_rps,

1072 - ramp_up_end_rps=args.ramp_up_end_rps,

1073 - ready_check_timeout_sec=args.ready_check_timeout_sec,

1074 - )

1144 + endpoint_type=args.endpoint_type,

1145 + api_url=api_url,

1146 + base_url=base_url,

1147 + model_id=model_id,

1148 + model_name=model_name,

1149 + tokenizer=tokenizer,

1150 + input_requests=input_requests,

1151 + logprobs=args.logprobs,

1152 + request_rate=args.request_rate,

1153 + burstiness=args.burstiness,

1154 + disable_tqdm=args.disable_tqdm,

1155 + profile=args.profile,

1156 + selected_percentile_metrics=args.percentile_metrics.split(","),

1157 + selected_percentiles=[

1158 + float(p) for p in args.metric_percentiles.split(",")

1159 + ],

1160 + ignore_eos=args.ignore_eos,

1161 + goodput_config_dict=goodput_config_dict,

1162 + max_concurrency=args.max_concurrency,

1163 + lora_modules=args.lora_modules,

1164 + extra_body=sampling_params,

1165 + ramp_up_strategy=args.ramp_up_strategy,

1166 + ramp_up_start_rps=args.ramp_up_start_rps,

1167 + ramp_up_end_rps=args.ramp_up_end_rps,

1168 + ready_check_timeout_sec=args.ready_check_timeout_sec,

1169 + )

1170

1171 # Save config and results to json

1172 result_json: dict[str, Any] = {}

@@ -1098,7 +1193,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1193

1194 # Traffic

1195 result_json["request_rate"] = (args.request_rate if args.request_rate

1101 - < float("inf") else "inf")

1196 + < float("inf") else "inf")

1197 result_json["burstiness"] = args.burstiness

1198 result_json["max_concurrency"] = args.max_concurrency

1199

@@ -1132,7 +1227,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1227 if args.max_concurrency is not None else "")

1228 label = label or endpoint_type

1229 if args.ramp_up_strategy is not None:

1135 - file_name = f"{label}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa

1230 + file_name = f"{label}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa

1231 else:

1232 file_name = f"{label}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa

1233 if args.result_filename:

@@ -1149,4 +1244,4 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1244 json.dump(result_json, outfile)

1245 save_to_pytorch_benchmark_format(args, result_json, file_name)

1246

1152 - return result_json

\ No newline at end of file

1247 + return result_json