A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
benchmark_serving support --served-model-name param (#12109)
Signed-off-by: zibai <zibai.gj@alibaba-inc.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
gujing committed
Jan 19, 2025 at 17:59 UTC
936db119ed390fadc7de448261226358e153e46c
2 files changed
+19
-3
benchmarks/backend_request_func.py
+6
-3
@@ -22,6 +22,7 @@ class RequestFuncInput:
22
prompt_len: int
23
output_len: int
24
model: str
25
+ model_name: Optional[str] = None
26
best_of: int = 1
27
logprobs: Optional[int] = None
28
extra_body: Optional[dict] = None
@@ -78,7 +79,7 @@ async def async_request_tgi(
79
continue
80
chunk_bytes = chunk_bytes.decode("utf-8")
81
81
- #NOTE: Sometimes TGI returns a ping response without
82
+ # NOTE: Sometimes TGI returns a ping response without
83
# any data, we should skip it.
84
if chunk_bytes.startswith(":"):
85
continue
@@ -235,7 +236,8 @@ async def async_request_openai_completions(
236
237
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
238
payload = {
238
- "model": request_func_input.model,
239
+ "model": request_func_input.model_name \
240
+ if request_func_input.model_name else request_func_input.model,
241
"prompt": request_func_input.prompt,
242
"temperature": 0.0,
243
"best_of": request_func_input.best_of,
@@ -328,7 +330,8 @@ async def async_request_openai_chat_completions(
330
if request_func_input.multi_modal_content:
331
content.append(request_func_input.multi_modal_content)
332
payload = {
331
- "model": request_func_input.model,
333
+ "model": request_func_input.model_name \
334
+ if request_func_input.model_name else request_func_input.model,
335
"messages": [
336
{
337
"role": "user",
benchmarks/benchmark_serving.py
+13
@@ -525,6 +525,7 @@ async def benchmark(
525
api_url: str,
526
base_url: str,
527
model_id: str,
528
+ model_name: str,
529
tokenizer: PreTrainedTokenizerBase,
530
input_requests: List[Tuple[str, int, int]],
531
logprobs: Optional[int],
@@ -553,6 +554,7 @@ async def benchmark(
554
"Multi-modal content is only supported on 'openai-chat' backend.")
555
test_input = RequestFuncInput(
556
model=model_id,
557
+ model_name=model_name,
558
prompt=test_prompt,
559
api_url=api_url,
560
prompt_len=test_prompt_len,
@@ -573,6 +575,7 @@ async def benchmark(
575
if profile:
576
print("Starting profiler...")
577
profile_input = RequestFuncInput(model=model_id,
578
+ model_name=model_name,
579
prompt=test_prompt,
580
api_url=base_url + "/start_profile",
581
prompt_len=test_prompt_len,
@@ -616,6 +619,7 @@ async def benchmark(
619
async for request in get_request(input_requests, request_rate, burstiness):
620
prompt, prompt_len, output_len, mm_content = request
621
request_func_input = RequestFuncInput(model=model_id,
622
+ model_name=model_name,
623
prompt=prompt,
624
api_url=api_url,
625
prompt_len=prompt_len,
@@ -780,6 +784,7 @@ def main(args: argparse.Namespace):
784
785
backend = args.backend
786
model_id = args.model
787
+ model_name = args.served_model_name
788
tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
789
tokenizer_mode = args.tokenizer_mode
790
@@ -877,6 +882,7 @@ def main(args: argparse.Namespace):
882
api_url=api_url,
883
base_url=base_url,
884
model_id=model_id,
885
+ model_name=model_name,
886
tokenizer=tokenizer,
887
input_requests=input_requests,
888
logprobs=args.logprobs,
@@ -1222,5 +1228,12 @@ if __name__ == "__main__":
1228
'always use the slow tokenizer. \n* '
1229
'"mistral" will always use the `mistral_common` tokenizer.')
1230
1231
+ parser.add_argument("--served-model-name",
1232
+ type=str,
1233
+ default=None,
1234
+ help="The model name used in the API. "
1235
+ "If not specified, the model name will be the "
1236
+ "same as the ``--model`` argument. ")
1237
+
1238
args = parser.parse_args()
1239
main(args)