@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a62bc0109c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Add Gamma-Distribution Request Generation Support for Serving Benchmark. (#10105)

Signed-off-by: Mozhou <spli161006@gmail.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Atlas committed Nov 7, 2024 at 19:20 UTC a62bc0109c3864b9dc770dc637e3acd332c730ea
1 file changed +52 -5
benchmarks/benchmark_serving.py
+52 -5
@@ -297,8 +297,33 @@ def sample_random_requests(
297 async def get_request(
298 input_requests: List[Tuple[str, int, int]],
299 request_rate: float,
300 + burstiness: float = 1.0,
301 ) -> AsyncGenerator[Tuple[str, int, int], None]:
302 + """
303 + Asynchronously generates requests at a specified rate
304 + with OPTIONAL burstiness.
305 +
306 + Args:
307 + input_requests:
308 + A list of input requests, each represented as a tuple.
309 + request_rate:
310 + The rate at which requests are generated (requests/s).
311 + burstiness (optional):
312 + The burstiness factor of the request generation.
313 + Only takes effect when request_rate is not inf.
314 + Default value is 1, which follows a Poisson process.
315 + Otherwise, the request intervals follow a gamma distribution.
316 + A lower burstiness value (0 < burstiness < 1) results
317 + in more bursty requests, while a higher burstiness value
318 + (burstiness > 1) results in a more uniform arrival of requests.
319 + """
320 input_requests = iter(input_requests)
321 +
322 + # Calculate scale parameter theta to maintain the desired request_rate.
323 + assert burstiness > 0, (
324 + f"A positive burstiness factor is expected, but given {burstiness}.")
325 + theta = 1.0 / (request_rate * burstiness)
326 +
327 for request in input_requests:
328 yield request
329
@@ -306,8 +331,9 @@ async def get_request(
331 # If the request rate is infinity, then we don't need to wait.
332 continue
333
309 - # Sample the request interval from the exponential distribution.
310 - interval = np.random.exponential(1.0 / request_rate)
334 + # Sample the request interval from the gamma distribution.
335 + # If burstiness is 1, it follows exponential distribution.
336 + interval = np.random.gamma(shape=burstiness, scale=theta)
337 # The next request will be sent after the interval.
338 await asyncio.sleep(interval)
339
@@ -426,6 +452,7 @@ async def benchmark(
452 logprobs: Optional[int],
453 best_of: int,
454 request_rate: float,
455 + burstiness: float,
456 disable_tqdm: bool,
457 profile: bool,
458 selected_percentile_metrics: List[str],
@@ -480,7 +507,13 @@ async def benchmark(
507 if profile_output.success:
508 print("Profiler started")
509
510 + if burstiness == 1.0:
511 + distribution = "Poisson process"
512 + else:
513 + distribution = "Gamma distribution"
514 +
515 print(f"Traffic request rate: {request_rate}")
516 + print(f"Burstiness factor: {burstiness} ({distribution})")
517 print(f"Maximum request concurrency: {max_concurrency}")
518
519 pbar = None if disable_tqdm else tqdm(total=len(input_requests))
@@ -502,7 +535,7 @@ async def benchmark(
535
536 benchmark_start_time = time.perf_counter()
537 tasks: List[asyncio.Task] = []
505 - async for request in get_request(input_requests, request_rate):
538 + async for request in get_request(input_requests, request_rate, burstiness):
539 prompt, prompt_len, output_len, mm_content = request
540 request_func_input = RequestFuncInput(model=model_id,
541 prompt=prompt,
@@ -769,6 +802,7 @@ def main(args: argparse.Namespace):
802 logprobs=args.logprobs,
803 best_of=args.best_of,
804 request_rate=args.request_rate,
805 + burstiness=args.burstiness,
806 disable_tqdm=args.disable_tqdm,
807 profile=args.profile,
808 selected_percentile_metrics=args.percentile_metrics.split(","),
@@ -807,6 +841,7 @@ def main(args: argparse.Namespace):
841 # Traffic
842 result_json["request_rate"] = (
843 args.request_rate if args.request_rate < float("inf") else "inf")
844 + result_json["burstiness"] = args.burstiness
845 result_json["max_concurrency"] = args.max_concurrency
846
847 # Merge with benchmark result
@@ -922,8 +957,20 @@ if __name__ == "__main__":
957 default=float("inf"),
958 help="Number of requests per second. If this is inf, "
959 "then all the requests are sent at time 0. "
925 - "Otherwise, we use Poisson process to synthesize "
926 - "the request arrival times.",
960 + "Otherwise, we use Poisson process or gamma distribution "
961 + "to synthesize the request arrival times.",
962 + )
963 + parser.add_argument(
964 + "--burstiness",
965 + type=float,
966 + default=1.0,
967 + help="Burstiness factor of the request generation. "
968 + "Only take effect when request_rate is not inf. "
969 + "Default value is 1, which follows Poisson process. "
970 + "Otherwise, the request intervals follow a gamma distribution. "
971 + "A lower burstiness value (0 < burstiness < 1) results in more "
972 + "bursty requests. A higher burstiness value (burstiness > 1) "
973 + "results in a more uniform arrival of requests.",
974 )
975 parser.add_argument("--seed", type=int, default=0)
976 parser.add_argument(