@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 936db119ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

benchmark_serving support --served-model-name param (#12109)

Signed-off-by: zibai <zibai.gj@alibaba-inc.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

gujing committed Jan 19, 2025 at 17:59 UTC 936db119ed390fadc7de448261226358e153e46c
2 files changed +19 -3
benchmarks/backend_request_func.py
+6 -3
@@ -22,6 +22,7 @@ class RequestFuncInput:
22 prompt_len: int
23 output_len: int
24 model: str
25 + model_name: Optional[str] = None
26 best_of: int = 1
27 logprobs: Optional[int] = None
28 extra_body: Optional[dict] = None
@@ -78,7 +79,7 @@ async def async_request_tgi(
79 continue
80 chunk_bytes = chunk_bytes.decode("utf-8")
81
81 - #NOTE: Sometimes TGI returns a ping response without
82 + # NOTE: Sometimes TGI returns a ping response without
83 # any data, we should skip it.
84 if chunk_bytes.startswith(":"):
85 continue
@@ -235,7 +236,8 @@ async def async_request_openai_completions(
236
237 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
238 payload = {
238 - "model": request_func_input.model,
239 + "model": request_func_input.model_name \
240 + if request_func_input.model_name else request_func_input.model,
241 "prompt": request_func_input.prompt,
242 "temperature": 0.0,
243 "best_of": request_func_input.best_of,
@@ -328,7 +330,8 @@ async def async_request_openai_chat_completions(
330 if request_func_input.multi_modal_content:
331 content.append(request_func_input.multi_modal_content)
332 payload = {
331 - "model": request_func_input.model,
333 + "model": request_func_input.model_name \
334 + if request_func_input.model_name else request_func_input.model,
335 "messages": [
336 {
337 "role": "user",
benchmarks/benchmark_serving.py
+13
@@ -525,6 +525,7 @@ async def benchmark(
525 api_url: str,
526 base_url: str,
527 model_id: str,
528 + model_name: str,
529 tokenizer: PreTrainedTokenizerBase,
530 input_requests: List[Tuple[str, int, int]],
531 logprobs: Optional[int],
@@ -553,6 +554,7 @@ async def benchmark(
554 "Multi-modal content is only supported on 'openai-chat' backend.")
555 test_input = RequestFuncInput(
556 model=model_id,
557 + model_name=model_name,
558 prompt=test_prompt,
559 api_url=api_url,
560 prompt_len=test_prompt_len,
@@ -573,6 +575,7 @@ async def benchmark(
575 if profile:
576 print("Starting profiler...")
577 profile_input = RequestFuncInput(model=model_id,
578 + model_name=model_name,
579 prompt=test_prompt,
580 api_url=base_url + "/start_profile",
581 prompt_len=test_prompt_len,
@@ -616,6 +619,7 @@ async def benchmark(
619 async for request in get_request(input_requests, request_rate, burstiness):
620 prompt, prompt_len, output_len, mm_content = request
621 request_func_input = RequestFuncInput(model=model_id,
622 + model_name=model_name,
623 prompt=prompt,
624 api_url=api_url,
625 prompt_len=prompt_len,
@@ -780,6 +784,7 @@ def main(args: argparse.Namespace):
784
785 backend = args.backend
786 model_id = args.model
787 + model_name = args.served_model_name
788 tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
789 tokenizer_mode = args.tokenizer_mode
790
@@ -877,6 +882,7 @@ def main(args: argparse.Namespace):
882 api_url=api_url,
883 base_url=base_url,
884 model_id=model_id,
885 + model_name=model_name,
886 tokenizer=tokenizer,
887 input_requests=input_requests,
888 logprobs=args.logprobs,
@@ -1222,5 +1228,12 @@ if __name__ == "__main__":
1228 'always use the slow tokenizer. \n* '
1229 '"mistral" will always use the `mistral_common` tokenizer.')
1230
1231 + parser.add_argument("--served-model-name",
1232 + type=str,
1233 + default=None,
1234 + help="The model name used in the API. "
1235 + "If not specified, the model name will be the "
1236 + "same as the ``--model`` argument. ")
1237 +
1238 args = parser.parse_args()
1239 main(args)