A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Fix] Move "model_config" as keyword args in chat_utils.py (#18098)
Signed-off-by: Linkun <github@lkchen.net>
lkchen committed
May 13, 2025 at 23:27 UTC
6685890d11abe4d919a176854b940d97fb9344e7
6 files changed
+42
-23
tests/entrypoints/openai/test_chat_template.py
+2
-2
@@ -122,10 +122,10 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
122
123
# Call the function and get the result
124
result = apply_hf_chat_template(
125
- model_config,
126
- tokenizer,
125
+ tokenizer=tokenizer,
126
conversation=mock_request.messages,
127
chat_template=mock_request.chat_template or template_content,
128
+ model_config=model_config,
129
tools=None,
130
add_generation_prompt=mock_request.add_generation_prompt,
131
continue_final_message=mock_request.continue_final_message,
tests/entrypoints/test_chat_utils.py
+5
-5
@@ -793,10 +793,10 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
793
)
794
795
vllm_result = apply_hf_chat_template(
796
- model_config,
797
- tokenizer,
796
+ tokenizer=tokenizer,
797
conversation=conversation,
798
chat_template=None,
799
+ model_config=model_config,
800
tools=None,
801
add_generation_prompt=True,
802
)
@@ -903,11 +903,11 @@ def test_resolve_content_format_hf_defined(model, expected_format):
903
print(_try_extract_ast(chat_template))
904
905
resolved_format = resolve_chat_template_content_format(
906
- model_config,
906
None, # Test detecting the tokenizer's chat_template
907
None,
908
"auto",
909
tokenizer,
910
+ model_config=model_config,
911
)
912
913
assert resolved_format == expected_format
@@ -962,11 +962,11 @@ def test_resolve_content_format_fallbacks(model, expected_format):
962
print(_try_extract_ast(chat_template))
963
964
resolved_format = resolve_chat_template_content_format(
965
- model_config,
965
None, # Test detecting the tokenizer's chat_template
966
None,
967
"auto",
968
tokenizer,
969
+ model_config=model_config,
970
)
971
972
assert resolved_format == expected_format
@@ -1021,11 +1021,11 @@ def test_resolve_content_format_examples(template_path, expected_format):
1021
print(_try_extract_ast(chat_template))
1022
1023
resolved_format = resolve_chat_template_content_format(
1024
- model_config,
1024
chat_template,
1025
None,
1026
"auto",
1027
dummy_tokenizer,
1028
+ model_config=model_config,
1029
)
1030
1031
assert resolved_format == expected_format
vllm/entrypoints/chat_utils.py
+27
-8
@@ -44,7 +44,7 @@ from vllm.transformers_utils.chat_templates import (
44
# yapf: enable
45
from vllm.transformers_utils.processor import cached_get_processor
46
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
47
-from vllm.utils import random_uuid
47
+from vllm.utils import deprecate_kwargs, random_uuid
48
49
logger = init_logger(__name__)
50
@@ -329,11 +329,17 @@ def resolve_mistral_chat_template(
329
"so it will be ignored.")
330
return None
331
332
+@deprecate_kwargs(
333
+ "trust_remote_code",
334
+ additional_message="Please use `model_config.trust_remote_code` instead.",
335
+)
336
def resolve_hf_chat_template(
333
- model_config: ModelConfig,
337
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
338
chat_template: Optional[str],
339
tools: Optional[list[dict[str, Any]]],
340
+ *,
341
+ model_config: ModelConfig,
342
+ trsut_remote_code: Optional[bool] = None,
343
) -> Optional[str]:
344
# 1st priority: The given chat template
345
if chat_template is not None:
@@ -379,18 +385,19 @@ def resolve_hf_chat_template(
385
386
387
def _resolve_chat_template_content_format(
382
- model_config: ModelConfig,
388
chat_template: Optional[str],
389
tools: Optional[list[dict[str, Any]]],
390
given_format: ChatTemplateContentFormatOption,
391
tokenizer: AnyTokenizer,
392
+ *,
393
+ model_config: ModelConfig,
394
) -> _ChatTemplateContentFormat:
395
if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
396
hf_chat_template = resolve_hf_chat_template(
390
- model_config,
397
tokenizer,
398
chat_template=chat_template,
399
tools=tools,
400
+ model_config=model_config,
401
)
402
else:
403
hf_chat_template = None
@@ -428,19 +435,25 @@ def _log_chat_template_content_format(
435
)
436
437
438
+@deprecate_kwargs(
439
+ "trust_remote_code",
440
+ additional_message="Please use `model_config.trust_remote_code` instead.",
441
+)
442
def resolve_chat_template_content_format(
432
- model_config: ModelConfig,
443
chat_template: Optional[str],
444
tools: Optional[list[dict[str, Any]]],
445
given_format: ChatTemplateContentFormatOption,
446
tokenizer: AnyTokenizer,
447
+ *,
448
+ model_config: ModelConfig,
449
+ trust_remote_code: Optional[bool] = None,
450
) -> _ChatTemplateContentFormat:
451
detected_format = _resolve_chat_template_content_format(
439
- model_config,
452
chat_template,
453
tools,
454
given_format,
455
tokenizer,
456
+ model_config=model_config,
457
)
458
459
_log_chat_template_content_format(
@@ -1191,21 +1204,27 @@ def parse_chat_messages_futures(
1204
return conversation, mm_tracker.all_mm_data()
1205
1206
1207
+@deprecate_kwargs(
1208
+ "trust_remote_code",
1209
+ additional_message="Please use `model_config.trust_remote_code` instead.",
1210
+)
1211
def apply_hf_chat_template(
1195
- model_config: ModelConfig,
1212
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1213
conversation: list[ConversationMessage],
1214
chat_template: Optional[str],
1215
tools: Optional[list[dict[str, Any]]],
1216
*,
1217
+ model_config: ModelConfig,
1218
tokenize: bool = False, # Different from HF's default
1219
+ # Deprecated, explicitly capture here so it doesn't slit into kwargs.
1220
+ trust_remote_code: Optional[bool] = None,
1221
**kwargs: Any,
1222
) -> str:
1223
hf_chat_template = resolve_hf_chat_template(
1205
- model_config,
1224
tokenizer,
1225
chat_template=chat_template,
1226
tools=tools,
1227
+ model_config=model_config,
1228
)
1229
1230
if hf_chat_template is None:
vllm/entrypoints/llm.py
+3
-3
@@ -731,11 +731,11 @@ class LLM:
731
tokenizer = self.get_tokenizer(lora_request)
732
model_config = self.llm_engine.get_model_config()
733
resolved_content_format = resolve_chat_template_content_format(
734
- model_config,
734
chat_template,
735
tools,
736
chat_template_content_format,
737
tokenizer,
738
+ model_config=model_config,
739
)
740
741
_chat_template_kwargs: dict[str, Any] = dict(
@@ -767,9 +767,9 @@ class LLM:
767
)
768
else:
769
prompt_str = apply_hf_chat_template(
770
- model_config,
771
- tokenizer,
770
+ tokenizer=tokenizer,
771
conversation=conversation,
772
+ model_config=model_config,
773
**_chat_template_kwargs,
774
)
775
# Special tokens are already included in chat templates so
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -971,10 +971,10 @@ async def init_app_state(
971
chat_template=resolved_chat_template)
972
else:
973
hf_chat_template = resolve_hf_chat_template(
974
- vllm_config.model_config,
975
- tokenizer,
974
+ tokenizer=tokenizer,
975
chat_template=None,
976
tools=None,
977
+ model_config=vllm_config.model_config,
978
)
979
980
if hf_chat_template != resolved_chat_template:
vllm/entrypoints/openai/serving_engine.py
+3
-3
@@ -670,11 +670,11 @@ class OpenAIServing:
670
model_config = self.model_config
671
672
resolved_content_format = resolve_chat_template_content_format(
673
- model_config,
673
chat_template,
674
tool_dicts,
675
chat_template_content_format,
676
tokenizer,
677
+ model_config=model_config,
678
)
679
conversation, mm_data_future = parse_chat_messages_futures(
680
messages,
@@ -701,9 +701,9 @@ class OpenAIServing:
701
)
702
else:
703
request_prompt = apply_hf_chat_template(
704
- model_config,
705
- tokenizer,
704
+ tokenizer=tokenizer,
705
conversation=conversation,
706
+ model_config=model_config,
707
**_chat_template_kwargs,
708
)
709