@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0dfba97b42
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] Fix multiple values for keyword argument error (#10075) (#10076)

Signed-off-by: Lei <ylxx@live.com>

Lei Yang committed Nov 7, 2024 at 17:07 UTC 0dfba97b42032987fd6bd3d304ac22dd314c89b1
1 file changed +11 -12
vllm/entrypoints/openai/serving_engine.py
+11 -12
@@ -443,29 +443,28 @@ class OpenAIServing:
443 tokenizer,
444 )
445
446 + _chat_template_kwargs: Dict[str, Any] = dict(
447 + chat_template=chat_template,
448 + add_generation_prompt=add_generation_prompt,
449 + continue_final_message=continue_final_message,
450 + tools=tool_dicts,
451 + documents=documents,
452 + )
453 + _chat_template_kwargs.update(chat_template_kwargs or {})
454 +
455 request_prompt: Union[str, List[int]]
456 is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
457 if is_mistral_tokenizer:
458 request_prompt = apply_mistral_chat_template(
459 tokenizer,
460 messages=messages,
452 - chat_template=chat_template,
453 - add_generation_prompt=add_generation_prompt,
454 - continue_final_message=continue_final_message,
455 - tools=tool_dicts,
456 - documents=documents,
457 - **(chat_template_kwargs or {}),
461 + **_chat_template_kwargs,
462 )
463 else:
464 request_prompt = apply_hf_chat_template(
465 tokenizer,
466 conversation=conversation,
463 - chat_template=chat_template,
464 - add_generation_prompt=add_generation_prompt,
465 - continue_final_message=continue_final_message,
466 - tools=tool_dicts,
467 - documents=documents,
468 - **(chat_template_kwargs or {}),
467 + **_chat_template_kwargs,
468 )
469
470 mm_data = await mm_data_future