A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
pass ignore_eos parameter to all benchmark_serving calls (#9349)
Grace Ho committed
Oct 15, 2024 at 13:30 UTC
5d264f4ab8d008f0ac5b7f0adb7189d70136f3ec
1 file changed
+18
-20
benchmarks/benchmark_serving.py
+18
-20
@@ -431,16 +431,15 @@ async def benchmark(
431
432
if profile:
433
print("Starting profiler...")
434
- profile_input = RequestFuncInput(
435
- model=model_id,
436
- prompt=test_prompt,
437
- api_url=base_url + "/start_profile",
438
- prompt_len=test_prompt_len,
439
- output_len=test_output_len,
440
- logprobs=logprobs,
441
- best_of=best_of,
442
- multi_modal_content=test_mm_content,
443
- )
434
+ profile_input = RequestFuncInput(model=model_id,
435
+ prompt=test_prompt,
436
+ api_url=base_url + "/start_profile",
437
+ prompt_len=test_prompt_len,
438
+ output_len=test_output_len,
439
+ logprobs=logprobs,
440
+ best_of=best_of,
441
+ multi_modal_content=test_mm_content,
442
+ ignore_eos=ignore_eos)
443
profile_output = await request_func(request_func_input=profile_input)
444
if profile_output.success:
445
print("Profiler started")
@@ -453,16 +452,15 @@ async def benchmark(
452
tasks: List[asyncio.Task] = []
453
async for request in get_request(input_requests, request_rate):
454
prompt, prompt_len, output_len, mm_content = request
456
- request_func_input = RequestFuncInput(
457
- model=model_id,
458
- prompt=prompt,
459
- api_url=api_url,
460
- prompt_len=prompt_len,
461
- output_len=output_len,
462
- logprobs=logprobs,
463
- best_of=best_of,
464
- multi_modal_content=mm_content,
465
- )
455
+ request_func_input = RequestFuncInput(model=model_id,
456
+ prompt=prompt,
457
+ api_url=api_url,
458
+ prompt_len=prompt_len,
459
+ output_len=output_len,
460
+ logprobs=logprobs,
461
+ best_of=best_of,
462
+ multi_modal_content=mm_content,
463
+ ignore_eos=ignore_eos)
464
tasks.append(
465
asyncio.create_task(
466
request_func(request_func_input=request_func_input,