@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d264f4ab8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

pass ignore_eos parameter to all benchmark_serving calls (#9349)

Grace Ho committed Oct 15, 2024 at 13:30 UTC 5d264f4ab8d008f0ac5b7f0adb7189d70136f3ec
1 file changed +18 -20
benchmarks/benchmark_serving.py
+18 -20
@@ -431,16 +431,15 @@ async def benchmark(
431
432 if profile:
433 print("Starting profiler...")
434 - profile_input = RequestFuncInput(
435 - model=model_id,
436 - prompt=test_prompt,
437 - api_url=base_url + "/start_profile",
438 - prompt_len=test_prompt_len,
439 - output_len=test_output_len,
440 - logprobs=logprobs,
441 - best_of=best_of,
442 - multi_modal_content=test_mm_content,
443 - )
434 + profile_input = RequestFuncInput(model=model_id,
435 + prompt=test_prompt,
436 + api_url=base_url + "/start_profile",
437 + prompt_len=test_prompt_len,
438 + output_len=test_output_len,
439 + logprobs=logprobs,
440 + best_of=best_of,
441 + multi_modal_content=test_mm_content,
442 + ignore_eos=ignore_eos)
443 profile_output = await request_func(request_func_input=profile_input)
444 if profile_output.success:
445 print("Profiler started")
@@ -453,16 +452,15 @@ async def benchmark(
452 tasks: List[asyncio.Task] = []
453 async for request in get_request(input_requests, request_rate):
454 prompt, prompt_len, output_len, mm_content = request
456 - request_func_input = RequestFuncInput(
457 - model=model_id,
458 - prompt=prompt,
459 - api_url=api_url,
460 - prompt_len=prompt_len,
461 - output_len=output_len,
462 - logprobs=logprobs,
463 - best_of=best_of,
464 - multi_modal_content=mm_content,
465 - )
455 + request_func_input = RequestFuncInput(model=model_id,
456 + prompt=prompt,
457 + api_url=api_url,
458 + prompt_len=prompt_len,
459 + output_len=output_len,
460 + logprobs=logprobs,
461 + best_of=best_of,
462 + multi_modal_content=mm_content,
463 + ignore_eos=ignore_eos)
464 tasks.append(
465 asyncio.create_task(
466 request_func(request_func_input=request_func_input,