@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e1837676a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[misc] Add LoRA to benchmark_serving (#12898)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Feb 8, 2025 at 14:45 UTC 7e1837676a3230b1b392d7699771cdb3f3407242
1 file changed +22 -2
benchmarks/benchmark_serving.py
+22 -2
@@ -537,6 +537,7 @@ async def benchmark(
537 ignore_eos: bool,
538 goodput_config_dict: Dict[str, float],
539 max_concurrency: Optional[int],
540 + lora_modules: Optional[List[str]],
541 ):
542 if backend in ASYNC_REQUEST_FUNCS:
543 request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -562,6 +563,7 @@ async def benchmark(
563 multi_modal_content=test_mm_content,
564 ignore_eos=ignore_eos,
565 )
566 +
567 test_output = await request_func(request_func_input=test_input)
568 if not test_output.success:
569 raise ValueError(
@@ -570,6 +572,11 @@ async def benchmark(
572 else:
573 print("Initial test run completed. Starting main benchmark run...")
574
575 + if lora_modules:
576 + # For each input request, choose a LoRA module at random.
577 + lora_modules = iter(
578 + [random.choice(lora_modules) for _ in range(len(input_requests))])
579 +
580 if profile:
581 print("Starting profiler...")
582 profile_input = RequestFuncInput(model=model_id,
@@ -616,8 +623,13 @@ async def benchmark(
623 tasks: List[asyncio.Task] = []
624 async for request in get_request(input_requests, request_rate, burstiness):
625 prompt, prompt_len, output_len, mm_content = request
619 - request_func_input = RequestFuncInput(model=model_id,
620 - model_name=model_name,
626 + req_model_id, req_model_name = model_id, model_name
627 + if lora_modules:
628 + req_lora_module = next(lora_modules)
629 + req_model_id, req_model_name = req_lora_module, req_lora_module
630 +
631 + request_func_input = RequestFuncInput(model=req_model_id,
632 + model_name=req_model_name,
633 prompt=prompt,
634 api_url=api_url,
635 prompt_len=prompt_len,
@@ -900,6 +912,7 @@ def main(args: argparse.Namespace):
912 ignore_eos=args.ignore_eos,
913 goodput_config_dict=goodput_config_dict,
914 max_concurrency=args.max_concurrency,
915 + lora_modules=args.lora_modules,
916 ))
917
918 # Save config and results to json
@@ -1237,5 +1250,12 @@ if __name__ == "__main__":
1250 "If not specified, the model name will be the "
1251 "same as the ``--model`` argument. ")
1252
1253 + parser.add_argument("--lora-modules",
1254 + nargs='+',
1255 + default=None,
1256 + help="A subset of LoRA module names passed in when "
1257 + "launching the server. For each request, the "
1258 + "script chooses a LoRA module at random.")
1259 +
1260 args = parser.parse_args()
1261 main(args)