A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Add Gamma-Distribution Request Generation Support for Serving Benchmark. (#10105)
Signed-off-by: Mozhou <spli161006@gmail.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Atlas committed
Nov 7, 2024 at 19:20 UTC
a62bc0109c3864b9dc770dc637e3acd332c730ea
1 file changed
+52
-5
benchmarks/benchmark_serving.py
+52
-5
@@ -297,8 +297,33 @@ def sample_random_requests(
297
async def get_request(
298
input_requests: List[Tuple[str, int, int]],
299
request_rate: float,
300
+ burstiness: float = 1.0,
301
) -> AsyncGenerator[Tuple[str, int, int], None]:
302
+ """
303
+ Asynchronously generates requests at a specified rate
304
+ with OPTIONAL burstiness.
305
+
306
+ Args:
307
+ input_requests:
308
+ A list of input requests, each represented as a tuple.
309
+ request_rate:
310
+ The rate at which requests are generated (requests/s).
311
+ burstiness (optional):
312
+ The burstiness factor of the request generation.
313
+ Only takes effect when request_rate is not inf.
314
+ Default value is 1, which follows a Poisson process.
315
+ Otherwise, the request intervals follow a gamma distribution.
316
+ A lower burstiness value (0 < burstiness < 1) results
317
+ in more bursty requests, while a higher burstiness value
318
+ (burstiness > 1) results in a more uniform arrival of requests.
319
+ """
320
input_requests = iter(input_requests)
321
+
322
+ # Calculate scale parameter theta to maintain the desired request_rate.
323
+ assert burstiness > 0, (
324
+ f"A positive burstiness factor is expected, but given {burstiness}.")
325
+ theta = 1.0 / (request_rate * burstiness)
326
+
327
for request in input_requests:
328
yield request
329
@@ -306,8 +331,9 @@ async def get_request(
331
# If the request rate is infinity, then we don't need to wait.
332
continue
333
309
- # Sample the request interval from the exponential distribution.
310
- interval = np.random.exponential(1.0 / request_rate)
334
+ # Sample the request interval from the gamma distribution.
335
+ # If burstiness is 1, it follows exponential distribution.
336
+ interval = np.random.gamma(shape=burstiness, scale=theta)
337
# The next request will be sent after the interval.
338
await asyncio.sleep(interval)
339
@@ -426,6 +452,7 @@ async def benchmark(
452
logprobs: Optional[int],
453
best_of: int,
454
request_rate: float,
455
+ burstiness: float,
456
disable_tqdm: bool,
457
profile: bool,
458
selected_percentile_metrics: List[str],
@@ -480,7 +507,13 @@ async def benchmark(
507
if profile_output.success:
508
print("Profiler started")
509
510
+ if burstiness == 1.0:
511
+ distribution = "Poisson process"
512
+ else:
513
+ distribution = "Gamma distribution"
514
+
515
print(f"Traffic request rate: {request_rate}")
516
+ print(f"Burstiness factor: {burstiness} ({distribution})")
517
print(f"Maximum request concurrency: {max_concurrency}")
518
519
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
@@ -502,7 +535,7 @@ async def benchmark(
535
536
benchmark_start_time = time.perf_counter()
537
tasks: List[asyncio.Task] = []
505
- async for request in get_request(input_requests, request_rate):
538
+ async for request in get_request(input_requests, request_rate, burstiness):
539
prompt, prompt_len, output_len, mm_content = request
540
request_func_input = RequestFuncInput(model=model_id,
541
prompt=prompt,
@@ -769,6 +802,7 @@ def main(args: argparse.Namespace):
802
logprobs=args.logprobs,
803
best_of=args.best_of,
804
request_rate=args.request_rate,
805
+ burstiness=args.burstiness,
806
disable_tqdm=args.disable_tqdm,
807
profile=args.profile,
808
selected_percentile_metrics=args.percentile_metrics.split(","),
@@ -807,6 +841,7 @@ def main(args: argparse.Namespace):
841
# Traffic
842
result_json["request_rate"] = (
843
args.request_rate if args.request_rate < float("inf") else "inf")
844
+ result_json["burstiness"] = args.burstiness
845
result_json["max_concurrency"] = args.max_concurrency
846
847
# Merge with benchmark result
@@ -922,8 +957,20 @@ if __name__ == "__main__":
957
default=float("inf"),
958
help="Number of requests per second. If this is inf, "
959
"then all the requests are sent at time 0. "
925
- "Otherwise, we use Poisson process to synthesize "
926
- "the request arrival times.",
960
+ "Otherwise, we use Poisson process or gamma distribution "
961
+ "to synthesize the request arrival times.",
962
+ )
963
+ parser.add_argument(
964
+ "--burstiness",
965
+ type=float,
966
+ default=1.0,
967
+ help="Burstiness factor of the request generation. "
968
+ "Only take effect when request_rate is not inf. "
969
+ "Default value is 1, which follows Poisson process. "
970
+ "Otherwise, the request intervals follow a gamma distribution. "
971
+ "A lower burstiness value (0 < burstiness < 1) results in more "
972
+ "bursty requests. A higher burstiness value (burstiness > 1) "
973
+ "results in a more uniform arrival of requests.",
974
)
975
parser.add_argument("--seed", type=int, default=0)
976
parser.add_argument(