A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend] Fix multiple values for keyword argument error (#10075) (#10076)
Signed-off-by: Lei <ylxx@live.com>
Lei Yang committed
Nov 7, 2024 at 17:07 UTC
0dfba97b42032987fd6bd3d304ac22dd314c89b1
1 file changed
+11
-12
vllm/entrypoints/openai/serving_engine.py
+11
-12
@@ -443,29 +443,28 @@ class OpenAIServing:
443
tokenizer,
444
)
445
446
+ _chat_template_kwargs: Dict[str, Any] = dict(
447
+ chat_template=chat_template,
448
+ add_generation_prompt=add_generation_prompt,
449
+ continue_final_message=continue_final_message,
450
+ tools=tool_dicts,
451
+ documents=documents,
452
+ )
453
+ _chat_template_kwargs.update(chat_template_kwargs or {})
454
+
455
request_prompt: Union[str, List[int]]
456
is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
457
if is_mistral_tokenizer:
458
request_prompt = apply_mistral_chat_template(
459
tokenizer,
460
messages=messages,
452
- chat_template=chat_template,
453
- add_generation_prompt=add_generation_prompt,
454
- continue_final_message=continue_final_message,
455
- tools=tool_dicts,
456
- documents=documents,
457
- **(chat_template_kwargs or {}),
461
+ **_chat_template_kwargs,
462
)
463
else:
464
request_prompt = apply_hf_chat_template(
465
tokenizer,
466
conversation=conversation,
463
- chat_template=chat_template,
464
- add_generation_prompt=add_generation_prompt,
465
- continue_final_message=continue_final_message,
466
- tools=tool_dicts,
467
- documents=documents,
468
- **(chat_template_kwargs or {}),
467
+ **_chat_template_kwargs,
468
)
469
470
mm_data = await mm_data_future