A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Misc][Benchmarking] Add variable request-rate ("ramp-up") to the benchmarking client. (#19423)
Signed-off-by: dtransposed <damian@damian-ml-machine.europe-west3-b.c.jetbrains-grazie.internal> Co-authored-by: dtransposed <damian@damian-ml-machine.europe-west3-b.c.jetbrains-grazie.internal> Co-authored-by: Roger Wang <hey@rogerw.me>
d.transposed committed
Jun 24, 2025 at 20:41 UTC
c635c5f7448f7ee5593c2ad7bb447b3aa1bc6d47
3 files changed
+330
-34
benchmarks/README.md
+15
@@ -269,6 +269,21 @@ python3 vllm/benchmarks/benchmark_serving.py \
269
--num-prompts 10
270
```
271
272
+### Running With Ramp-Up Request Rate
273
+
274
+The benchmark tool also supports ramping up the request rate over the
275
+duration of the benchmark run. This can be useful for stress testing the
276
+server or finding the maximum throughput that it can handle, given some latency budget.
277
+
278
+Two ramp-up strategies are supported:
279
+- `linear`: Increases the request rate linearly from a start value to an end value.
280
+- `exponential`: Increases the request rate exponentially.
281
+
282
+The following arguments can be used to control the ramp-up:
283
+- `--ramp-up-strategy`: The ramp-up strategy to use (`linear` or `exponential`).
284
+- `--ramp-up-start-rps`: The request rate at the beginning of the benchmark.
285
+- `--ramp-up-end-rps`: The request rate at the end of the benchmark.
286
+
287
---
288
## Example - Offline Throughput Benchmark
289
benchmarks/benchmark_serving.py
+161
-17
@@ -33,7 +33,7 @@ import warnings
33
from collections.abc import AsyncGenerator, Iterable
34
from dataclasses import dataclass
35
from datetime import datetime
36
-from typing import Any, Optional
36
+from typing import Any, Literal, Optional
37
38
import numpy as np
39
from tqdm.asyncio import tqdm
@@ -107,14 +107,42 @@ class BenchmarkMetrics:
107
percentiles_e2el_ms: list[tuple[float, float]]
108
109
110
+def _get_current_request_rate(
111
+ ramp_up_strategy: Optional[Literal["linear", "exponential"]],
112
+ ramp_up_start_rps: Optional[int],
113
+ ramp_up_end_rps: Optional[int],
114
+ request_index: int,
115
+ total_requests: int,
116
+ request_rate: float,
117
+) -> float:
118
+ if (
119
+ ramp_up_strategy
120
+ and ramp_up_start_rps is not None
121
+ and ramp_up_end_rps is not None
122
+ ):
123
+ progress = request_index / max(total_requests - 1, 1)
124
+ if ramp_up_strategy == "linear":
125
+ increase = (ramp_up_end_rps - ramp_up_start_rps) * progress
126
+ return ramp_up_start_rps + increase
127
+ elif ramp_up_strategy == "exponential":
128
+ ratio = ramp_up_end_rps / ramp_up_start_rps
129
+ return ramp_up_start_rps * (ratio**progress)
130
+ else:
131
+ raise ValueError(f"Unknown ramp-up strategy: {ramp_up_strategy}")
132
+ return request_rate
133
+
134
+
135
async def get_request(
136
input_requests: list[SampleRequest],
137
request_rate: float,
138
burstiness: float = 1.0,
114
-) -> AsyncGenerator[SampleRequest, None]:
139
+ ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
140
+ ramp_up_start_rps: Optional[int] = None,
141
+ ramp_up_end_rps: Optional[int] = None,
142
+) -> AsyncGenerator[tuple[SampleRequest, float], None]:
143
"""
144
Asynchronously generates requests at a specified rate
117
- with OPTIONAL burstiness.
145
+ with OPTIONAL burstiness and OPTIONAL ramp-up strategy.
146
147
Args:
148
input_requests:
@@ -129,22 +157,44 @@ async def get_request(
157
A lower burstiness value (0 < burstiness < 1) results
158
in more bursty requests, while a higher burstiness value
159
(burstiness > 1) results in a more uniform arrival of requests.
160
+ ramp_up_strategy (optional):
161
+ The ramp-up strategy. Can be "linear" or "exponential".
162
+ If None, uses constant request rate (specified by request_rate).
163
+ ramp_up_start_rps (optional):
164
+ The starting request rate for ramp-up.
165
+ ramp_up_end_rps (optional):
166
+ The ending request rate for ramp-up.
167
"""
133
- input_requests: Iterable[SampleRequest] = iter(input_requests)
134
-
135
- # Calculate scale parameter theta to maintain the desired request_rate.
168
assert burstiness > 0, (
169
f"A positive burstiness factor is expected, but given {burstiness}."
170
)
139
- theta = 1.0 / (request_rate * burstiness)
171
+ # Convert to list to get length for ramp-up calculations
172
+ if isinstance(input_requests, Iterable) and not isinstance(input_requests, list):
173
+ input_requests = list(input_requests)
174
+
175
+ total_requests = len(input_requests)
176
+ request_index = 0
177
178
for request in input_requests:
142
- yield request
179
+ current_request_rate = _get_current_request_rate(
180
+ ramp_up_strategy,
181
+ ramp_up_start_rps,
182
+ ramp_up_end_rps,
183
+ request_index,
184
+ total_requests,
185
+ request_rate,
186
+ )
187
+
188
+ yield request, current_request_rate
189
144
- if request_rate == float("inf"):
190
+ request_index += 1
191
+
192
+ if current_request_rate == float("inf"):
193
# If the request rate is infinity, then we don't need to wait.
194
continue
195
196
+ theta = 1.0 / (current_request_rate * burstiness)
197
+
198
# Sample the request interval from the gamma distribution.
199
# If burstiness is 1, it follows exponential distribution.
200
interval = np.random.gamma(shape=burstiness, scale=theta)
@@ -290,6 +340,9 @@ async def benchmark(
340
max_concurrency: Optional[int],
341
lora_modules: Optional[Iterable[str]],
342
extra_body: Optional[dict],
343
+ ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
344
+ ramp_up_start_rps: Optional[int] = None,
345
+ ramp_up_end_rps: Optional[int] = None,
346
):
347
if backend in ASYNC_REQUEST_FUNCS:
348
request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -353,7 +406,15 @@ async def benchmark(
406
407
distribution = "Poisson process" if burstiness == 1.0 else "Gamma distribution"
408
356
- print(f"Traffic request rate: {request_rate}")
409
+ if ramp_up_strategy is not None:
410
+ print(
411
+ f"Traffic ramp-up strategy: {ramp_up_strategy}. Will increase "
412
+ f"RPS from {ramp_up_start_rps} to {ramp_up_end_rps} RPS over "
413
+ "the duration of the benchmark."
414
+ )
415
+ else:
416
+ print(f"Traffic request rate: {request_rate} RPS.")
417
+
418
print(f"Burstiness factor: {burstiness} ({distribution})")
419
print(f"Maximum request concurrency: {max_concurrency}")
420
@@ -373,7 +434,34 @@ async def benchmark(
434
435
benchmark_start_time = time.perf_counter()
436
tasks: list[asyncio.Task] = []
376
- async for request in get_request(input_requests, request_rate, burstiness):
437
+
438
+ rps_change_events = []
439
+ last_int_rps = -1
440
+ if ramp_up_strategy is not None and ramp_up_start_rps is not None:
441
+ last_int_rps = ramp_up_start_rps
442
+ rps_change_events.append(
443
+ {
444
+ "rps": last_int_rps,
445
+ "timestamp": datetime.now().isoformat(),
446
+ }
447
+ )
448
+
449
+ async for request, current_request_rate in get_request(
450
+ input_requests,
451
+ request_rate,
452
+ burstiness,
453
+ ramp_up_strategy,
454
+ ramp_up_start_rps,
455
+ ramp_up_end_rps,
456
+ ):
457
+ if ramp_up_strategy is not None:
458
+ current_int_rps = int(current_request_rate)
459
+ if current_int_rps > last_int_rps:
460
+ timestamp = datetime.now().isoformat()
461
+ for rps_val in range(last_int_rps + 1, current_int_rps + 1):
462
+ rps_change_events.append({"rps": rps_val, "timestamp": timestamp})
463
+ last_int_rps = current_int_rps
464
+
465
prompt, prompt_len, output_len, mm_content = (
466
request.prompt,
467
request.prompt_len,
@@ -397,11 +485,8 @@ async def benchmark(
485
ignore_eos=ignore_eos,
486
extra_body=extra_body,
487
)
400
- tasks.append(
401
- asyncio.create_task(
402
- limited_request_func(request_func_input=request_func_input, pbar=pbar)
403
- )
404
- )
488
+ task = limited_request_func(request_func_input=request_func_input, pbar=pbar)
489
+ tasks.append(asyncio.create_task(task))
490
outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)
491
492
if profile:
@@ -477,6 +562,9 @@ async def benchmark(
562
"errors": [output.error for output in outputs],
563
}
564
565
+ if rps_change_events:
566
+ result["rps_change_events"] = rps_change_events
567
+
568
def process_one_metric(
569
# E.g., "ttft"
570
metric_attribute_name: str,
@@ -610,6 +698,26 @@ def main(args: argparse.Namespace):
698
tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
699
tokenizer_mode = args.tokenizer_mode
700
701
+ # Validate ramp-up arguments
702
+ if args.ramp_up_strategy is not None:
703
+ if args.request_rate != float("inf"):
704
+ raise ValueError(
705
+ "When using ramp-up, do not specify --request-rate. "
706
+ "The request rate will be controlled by ramp-up parameters. "
707
+ "Please remove the --request-rate argument."
708
+ )
709
+ if args.ramp_up_start_rps is None or args.ramp_up_end_rps is None:
710
+ raise ValueError(
711
+ "When using --ramp-up-strategy, both --ramp-up-start-rps and "
712
+ "--ramp-up-end-rps must be specified"
713
+ )
714
+ if args.ramp_up_start_rps < 0 or args.ramp_up_end_rps < 0:
715
+ raise ValueError("Ramp-up start and end RPS must be non-negative")
716
+ if args.ramp_up_start_rps > args.ramp_up_end_rps:
717
+ raise ValueError("Ramp-up start RPS must be less than end RPS")
718
+ if args.ramp_up_strategy == "exponential" and args.ramp_up_start_rps == 0:
719
+ raise ValueError("For exponential ramp-up, the start RPS cannot be 0.")
720
+
721
if args.base_url is not None:
722
api_url = f"{args.base_url}{args.endpoint}"
723
base_url = f"{args.base_url}"
@@ -802,6 +910,9 @@ def main(args: argparse.Namespace):
910
max_concurrency=args.max_concurrency,
911
lora_modules=args.lora_modules,
912
extra_body=sampling_params,
913
+ ramp_up_strategy=args.ramp_up_strategy,
914
+ ramp_up_start_rps=args.ramp_up_start_rps,
915
+ ramp_up_end_rps=args.ramp_up_end_rps,
916
)
917
)
918
@@ -834,6 +945,11 @@ def main(args: argparse.Namespace):
945
result_json["burstiness"] = args.burstiness
946
result_json["max_concurrency"] = args.max_concurrency
947
948
+ if args.ramp_up_strategy is not None:
949
+ result_json["ramp_up_strategy"] = args.ramp_up_strategy
950
+ result_json["ramp_up_start_rps"] = args.ramp_up_start_rps
951
+ result_json["ramp_up_end_rps"] = args.ramp_up_end_rps
952
+
953
# Merge with benchmark result
954
result_json = {**result_json, **benchmark_result}
955
@@ -859,7 +975,10 @@ def main(args: argparse.Namespace):
975
if args.max_concurrency is not None
976
else ""
977
)
862
- file_name = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
978
+ if args.ramp_up_strategy is not None:
979
+ file_name = f"{backend}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
980
+ else:
981
+ file_name = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
982
if args.result_filename:
983
file_name = args.result_filename
984
if args.result_dir:
@@ -1225,6 +1344,31 @@ def create_argument_parser():
1344
"script chooses a LoRA module at random.",
1345
)
1346
1347
+ parser.add_argument(
1348
+ "--ramp-up-strategy",
1349
+ type=str,
1350
+ default=None,
1351
+ choices=["linear", "exponential"],
1352
+ help="The ramp-up strategy. This would be used to "
1353
+ "ramp up the request rate from initial RPS to final "
1354
+ "RPS rate (specified by --ramp-up-start-rps and --ramp-up-end-rps). "
1355
+ "over the duration of the benchmark.",
1356
+ )
1357
+ parser.add_argument(
1358
+ "--ramp-up-start-rps",
1359
+ type=int,
1360
+ default=None,
1361
+ help="The starting request rate for ramp-up (RPS). "
1362
+ "Needs to be specified when --ramp-up-strategy is used.",
1363
+ )
1364
+ parser.add_argument(
1365
+ "--ramp-up-end-rps",
1366
+ type=int,
1367
+ default=None,
1368
+ help="The ending request rate for ramp-up (RPS). "
1369
+ "Needs to be specified when --ramp-up-strategy is used.",
1370
+ )
1371
+
1372
return parser
1373
1374
vllm/benchmarks/serve.py
+154
-17
@@ -26,7 +26,7 @@ import warnings
26
from collections.abc import AsyncGenerator, Iterable
27
from dataclasses import dataclass
28
from datetime import datetime
29
-from typing import Any, Optional
29
+from typing import Any, Literal, Optional
30
31
import numpy as np
32
from tqdm.asyncio import tqdm
@@ -75,14 +75,39 @@ class BenchmarkMetrics:
75
percentiles_e2el_ms: list[tuple[float, float]]
76
77
78
+def _get_current_request_rate(
79
+ ramp_up_strategy: Optional[Literal["linear", "exponential"]],
80
+ ramp_up_start_rps: Optional[int],
81
+ ramp_up_end_rps: Optional[int],
82
+ request_index: int,
83
+ total_requests: int,
84
+ request_rate: float,
85
+) -> float:
86
+ if (ramp_up_strategy and ramp_up_start_rps is not None
87
+ and ramp_up_end_rps is not None):
88
+ progress = request_index / max(total_requests - 1, 1)
89
+ if ramp_up_strategy == "linear":
90
+ increase = (ramp_up_end_rps - ramp_up_start_rps) * progress
91
+ return ramp_up_start_rps + increase
92
+ elif ramp_up_strategy == "exponential":
93
+ ratio = ramp_up_end_rps / ramp_up_start_rps
94
+ return ramp_up_start_rps * (ratio**progress)
95
+ else:
96
+ raise ValueError(f"Unknown ramp-up strategy: {ramp_up_strategy}")
97
+ return request_rate
98
+
99
+
100
async def get_request(
101
input_requests: list[SampleRequest],
102
request_rate: float,
103
burstiness: float = 1.0,
82
-) -> AsyncGenerator[SampleRequest, None]:
104
+ ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
105
+ ramp_up_start_rps: Optional[int] = None,
106
+ ramp_up_end_rps: Optional[int] = None,
107
+) -> AsyncGenerator[tuple[SampleRequest, float], None]:
108
"""
109
Asynchronously generates requests at a specified rate
85
- with OPTIONAL burstiness.
110
+ with OPTIONAL burstiness and OPTIONAL ramp-up strategy.
111
112
Args:
113
input_requests:
@@ -97,21 +122,42 @@ async def get_request(
122
A lower burstiness value (0 < burstiness < 1) results
123
in more bursty requests, while a higher burstiness value
124
(burstiness > 1) results in a more uniform arrival of requests.
125
+ ramp_up_strategy (optional):
126
+ The ramp-up strategy. Can be "linear" or "exponential".
127
+ If None, uses constant request rate (specified by request_rate).
128
+ ramp_up_start_rps (optional):
129
+ The starting request rate for ramp-up.
130
+ ramp_up_end_rps (optional):
131
+ The ending request rate for ramp-up.
132
"""
101
- input_requests: Iterable[SampleRequest] = iter(input_requests)
102
-
103
- # Calculate scale parameter theta to maintain the desired request_rate.
133
assert burstiness > 0, (
134
f"A positive burstiness factor is expected, but given {burstiness}.")
106
- theta = 1.0 / (request_rate * burstiness)
135
+ # Convert to list to get length for ramp-up calculations
136
+ if isinstance(input_requests, Iterable) and not isinstance(
137
+ input_requests, list):
138
+ input_requests = list(input_requests)
139
108
- for request in input_requests:
109
- yield request
140
+ total_requests = len(input_requests)
141
+ request_index = 0
142
111
- if request_rate == float("inf"):
143
+ for request in input_requests:
144
+ current_request_rate = _get_current_request_rate(ramp_up_strategy,
145
+ ramp_up_start_rps,
146
+ ramp_up_end_rps,
147
+ request_index,
148
+ total_requests,
149
+ request_rate)
150
+
151
+ yield request, current_request_rate
152
+
153
+ request_index += 1
154
+
155
+ if current_request_rate == float("inf"):
156
# If the request rate is infinity, then we don't need to wait.
157
continue
158
159
+ theta = 1.0 / (current_request_rate * burstiness)
160
+
161
# Sample the request interval from the gamma distribution.
162
# If burstiness is 1, it follows exponential distribution.
163
interval = np.random.gamma(shape=burstiness, scale=theta)
@@ -259,6 +305,9 @@ async def benchmark(
305
max_concurrency: Optional[int],
306
lora_modules: Optional[Iterable[str]],
307
extra_body: Optional[dict],
308
+ ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
309
+ ramp_up_start_rps: Optional[int] = None,
310
+ ramp_up_end_rps: Optional[int] = None,
311
):
312
if endpoint_type in ASYNC_REQUEST_FUNCS:
313
request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
@@ -316,12 +365,16 @@ async def benchmark(
365
if profile_output.success:
366
print("Profiler started")
367
319
- if burstiness == 1.0:
320
- distribution = "Poisson process"
368
+ distribution = ("Poisson process" if burstiness == 1.0
369
+ else "Gamma distribution")
370
+
371
+ if ramp_up_strategy is not None:
372
+ print(f"Traffic ramp-up strategy: {ramp_up_strategy}.")
373
+ print(f"Will increase RPS from {ramp_up_start_rps} to "
374
+ f"{ramp_up_end_rps} RPS over the duration of the benchmark.")
375
else:
322
- distribution = "Gamma distribution"
376
+ print(f"Traffic request rate: {request_rate}")
377
324
- print(f"Traffic request rate: {request_rate}")
378
print(f"Burstiness factor: {burstiness} ({distribution})")
379
print(f"Maximum request concurrency: {max_concurrency}")
380
@@ -344,7 +397,29 @@ async def benchmark(
397
398
benchmark_start_time = time.perf_counter()
399
tasks: list[asyncio.Task] = []
347
- async for request in get_request(input_requests, request_rate, burstiness):
400
+
401
+ rps_change_events = []
402
+ last_int_rps = -1
403
+ if ramp_up_strategy is not None and ramp_up_start_rps is not None:
404
+ last_int_rps = ramp_up_start_rps
405
+ rps_change_events.append({
406
+ "rps": last_int_rps,
407
+ "timestamp": datetime.now().isoformat(),
408
+ })
409
+
410
+ async for request, current_request_rate in get_request(
411
+ input_requests, request_rate, burstiness, ramp_up_strategy,
412
+ ramp_up_start_rps, ramp_up_end_rps):
413
+ if ramp_up_strategy is not None:
414
+ current_int_rps = int(current_request_rate)
415
+ if current_int_rps > last_int_rps:
416
+ timestamp = datetime.now().isoformat()
417
+ for rps_val in range(last_int_rps + 1, current_int_rps + 1):
418
+ rps_change_events.append({
419
+ "rps": rps_val,
420
+ "timestamp": timestamp
421
+ })
422
+ last_int_rps = current_int_rps
423
prompt, prompt_len, output_len, mm_content = (
424
request.prompt,
425
request.prompt_len,
@@ -435,6 +510,9 @@ async def benchmark(
510
"errors": [output.error for output in outputs],
511
}
512
513
+ if rps_change_events:
514
+ result["rps_change_events"] = rps_change_events
515
+
516
def process_one_metric(
517
# E.g., "ttft"
518
metric_attribute_name: str,
@@ -771,12 +849,60 @@ def add_cli_args(parser: argparse.ArgumentParser):
849
"launching the server. For each request, the "
850
"script chooses a LoRA module at random.")
851
852
+ parser.add_argument(
853
+ "--ramp-up-strategy",
854
+ type=str,
855
+ default=None,
856
+ choices=["linear", "exponential"],
857
+ help="The ramp-up strategy. This would be used to "
858
+ "ramp up the request rate from initial RPS to final "
859
+ "RPS rate (specified by --ramp-up-start-rps and "
860
+ "--ramp-up-end-rps.) over the duration of the benchmark."
861
+ )
862
+ parser.add_argument(
863
+ "--ramp-up-start-rps",
864
+ type=int,
865
+ default=None,
866
+ help="The starting request rate for ramp-up (RPS). "
867
+ "Needs to be specified when --ramp-up-strategy is used.",
868
+ )
869
+ parser.add_argument(
870
+ "--ramp-up-end-rps",
871
+ type=int,
872
+ default=None,
873
+ help="The ending request rate for ramp-up (RPS). "
874
+ "Needs to be specified when --ramp-up-strategy is used.",
875
+ )
876
+
877
878
def main(args: argparse.Namespace):
879
print(args)
880
random.seed(args.seed)
881
np.random.seed(args.seed)
882
883
+ # Validate ramp-up arguments
884
+ if args.ramp_up_strategy is not None:
885
+ if args.request_rate != float("inf"):
886
+ raise ValueError(
887
+ "When using ramp-up, do not specify --request-rate. "
888
+ "The request rate will be controlled by ramp-up parameters. "
889
+ "Please remove the --request-rate argument."
890
+ )
891
+ if args.ramp_up_start_rps is None or args.ramp_up_end_rps is None:
892
+ raise ValueError(
893
+ "When using --ramp-up-strategy, both --ramp-up-start-rps and "
894
+ "--ramp-up-end-rps must be specified"
895
+ )
896
+ if args.ramp_up_start_rps < 0 or args.ramp_up_end_rps < 0:
897
+ raise ValueError("Ramp-up start and end RPS must be non-negative")
898
+ if args.ramp_up_start_rps > args.ramp_up_end_rps:
899
+ raise ValueError("Ramp-up start RPS must be less than end RPS")
900
+ if (args.ramp_up_strategy == "exponential"
901
+ and args.ramp_up_start_rps == 0):
902
+ raise ValueError(
903
+ "For exponential ramp-up, the start RPS cannot be 0.")
904
+
905
+ endpoint_type = args.endpoint_type
906
label = args.label
907
model_id = args.model
908
model_name = args.served_model_name
@@ -849,6 +975,9 @@ def main(args: argparse.Namespace):
975
max_concurrency=args.max_concurrency,
976
lora_modules=args.lora_modules,
977
extra_body=sampling_params,
978
+ ramp_up_strategy=args.ramp_up_strategy,
979
+ ramp_up_start_rps=args.ramp_up_start_rps,
980
+ ramp_up_end_rps=args.ramp_up_end_rps,
981
))
982
983
# Save config and results to json
@@ -881,6 +1010,11 @@ def main(args: argparse.Namespace):
1010
result_json["burstiness"] = args.burstiness
1011
result_json["max_concurrency"] = args.max_concurrency
1012
1013
+ if args.ramp_up_strategy is not None:
1014
+ result_json["ramp_up_strategy"] = args.ramp_up_strategy
1015
+ result_json["ramp_up_start_rps"] = args.ramp_up_start_rps
1016
+ result_json["ramp_up_end_rps"] = args.ramp_up_end_rps
1017
+
1018
# Merge with benchmark result
1019
result_json = {**result_json, **benchmark_result}
1020
@@ -903,8 +1037,11 @@ def main(args: argparse.Namespace):
1037
base_model_id = model_id.split("/")[-1]
1038
max_concurrency_str = (f"-concurrency{args.max_concurrency}"
1039
if args.max_concurrency is not None else "")
906
- label = label or args.endpoint_type
907
- file_name = f"{label}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" #noqa
1040
+ label = label or endpoint_type
1041
+ if args.ramp_up_strategy is not None:
1042
+ file_name = f"{label}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1043
+ else:
1044
+ file_name = f"{label}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1045
if args.result_filename:
1046
file_name = args.result_filename
1047
if args.result_dir: