A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Add filtering for chat template kwargs (#25794)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Russell Bryant committed
Sep 27, 2025 at 06:46 UTC
7977e5027c2250a4abc1f474c5619c40b4e5682f
5 files changed
+158
-6
tests/entrypoints/test_chat_utils.py
+85
@@ -19,6 +19,7 @@ from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
19
parse_chat_messages,
20
parse_chat_messages_futures,
21
resolve_chat_template_content_format,
22
+ resolve_chat_template_kwargs,
23
resolve_hf_chat_template)
24
from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
25
from vllm.multimodal.utils import (encode_audio_base64, encode_image_base64,
@@ -37,6 +38,7 @@ QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
38
QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
39
QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
40
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
41
+QWEN3_MODEL_ID = "Qwen/Qwen3-8B"
42
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
43
HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
44
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
@@ -2255,6 +2257,89 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
2257
assert isinstance(chat_template, str)
2258
2259
2260
+@pytest.mark.parametrize(
2261
+ "model, expected_kwargs",
2262
+ [
2263
+ (
2264
+ QWEN2VL_MODEL_ID,
2265
+ {
2266
+ "add_vision_id", "add_generation_prompt",
2267
+ "continue_final_message", "tools"
2268
+ },
2269
+ ),
2270
+ (
2271
+ QWEN3_MODEL_ID,
2272
+ {
2273
+ "enable_thinking", "add_generation_prompt",
2274
+ "continue_final_message", "tools"
2275
+ },
2276
+ ),
2277
+ ],
2278
+)
2279
+def test_resolve_hf_chat_template_kwargs(sample_json_schema, model,
2280
+ expected_kwargs):
2281
+ """checks that chat_template is a dict type for HF models."""
2282
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
2283
+ model_info.check_available_online(on_fail="skip")
2284
+
2285
+ tools = ([{
2286
+ "type": "function",
2287
+ "function": {
2288
+ "name": "dummy_function_name",
2289
+ "description": "This is a dummy function",
2290
+ "parameters": sample_json_schema,
2291
+ },
2292
+ }])
2293
+
2294
+ chat_template_kwargs = {
2295
+ # both unused
2296
+ "unsed_kwargs_1": 123,
2297
+ "unsed_kwargs_2": "abc",
2298
+ # should not appear
2299
+ "chat_template": "{% Hello world! %}",
2300
+ # used by tokenizer
2301
+ "continue_final_message": True,
2302
+ "tools": tools,
2303
+ # both used by Qwen2-VL and Qwen3
2304
+ "add_generation_prompt": True,
2305
+ # only used by Qwen2-VL
2306
+ "add_vision_id": True,
2307
+ # only used by Qwen3
2308
+ "enable_thinking": True,
2309
+ }
2310
+
2311
+ model_config = ModelConfig(
2312
+ model,
2313
+ tokenizer=model_info.tokenizer or model,
2314
+ tokenizer_mode=model_info.tokenizer_mode,
2315
+ revision=model_info.revision,
2316
+ trust_remote_code=model_info.trust_remote_code,
2317
+ hf_overrides=model_info.hf_overrides,
2318
+ skip_tokenizer_init=model_info.skip_tokenizer_init,
2319
+ enforce_eager=model_info.enforce_eager,
2320
+ dtype=model_info.dtype)
2321
+
2322
+ # Build the tokenizer
2323
+ tokenizer = get_tokenizer(
2324
+ model,
2325
+ trust_remote_code=model_config.trust_remote_code,
2326
+ )
2327
+
2328
+ # Test detecting the tokenizer's chat_template
2329
+ chat_template = resolve_hf_chat_template(
2330
+ tokenizer,
2331
+ chat_template=None,
2332
+ tools=tools,
2333
+ model_config=model_config,
2334
+ )
2335
+ resolved_chat_template_kwargs = resolve_chat_template_kwargs(
2336
+ tokenizer,
2337
+ chat_template=chat_template,
2338
+ chat_template_kwargs=chat_template_kwargs,
2339
+ )
2340
+ assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs
2341
+
2342
+
2343
# NOTE: Qwen2-Audio default chat template is specially defined inside
2344
# processor class instead of using `tokenizer_config.json`
2345
# yapf: disable
vllm/entrypoints/chat_utils.py
+52
-2
@@ -11,7 +11,12 @@ from pathlib import Path
11
from typing import (Any, Callable, Generic, Literal, Optional, TypeVar, Union,
12
cast)
13
14
+import jinja2
15
+import jinja2.ext
16
+import jinja2.meta
17
import jinja2.nodes
18
+import jinja2.parser
19
+import jinja2.sandbox
20
import transformers.utils.chat_template_utils as hf_chat_utils
21
# yapf conflicts with isort for this block
22
# yapf: disable
@@ -50,7 +55,7 @@ from vllm.transformers_utils.chat_templates import (
55
# yapf: enable
56
from vllm.transformers_utils.processor import cached_get_processor
57
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
53
-from vllm.utils import random_uuid
58
+from vllm.utils import random_uuid, supports_kw
59
60
logger = init_logger(__name__)
61
@@ -1554,6 +1559,46 @@ def parse_chat_messages_futures(
1559
return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1560
1561
1562
+# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412
1563
+# only preserve the parse function used to resolve chat template kwargs
1564
+class AssistantTracker(jinja2.ext.Extension):
1565
+ tags = {"generation"}
1566
+
1567
+ def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.CallBlock:
1568
+ lineno = next(parser.stream).lineno
1569
+ body = parser.parse_statements(["name:endgeneration"], drop_needle=True)
1570
+ call = self.call_method("_generation_support")
1571
+ call_block = jinja2.nodes.CallBlock(call, [], [], body)
1572
+ return call_block.set_lineno(lineno)
1573
+
1574
+
1575
+def resolve_chat_template_kwargs(
1576
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1577
+ chat_template: str,
1578
+ chat_template_kwargs: dict[str, Any],
1579
+) -> dict[str, Any]:
1580
+ fn_kw = {
1581
+ k for k in chat_template_kwargs
1582
+ if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False)
1583
+ }
1584
+
1585
+ env = jinja2.sandbox.ImmutableSandboxedEnvironment(
1586
+ trim_blocks=True,
1587
+ lstrip_blocks=True,
1588
+ extensions=[AssistantTracker, jinja2.ext.loopcontrols],
1589
+ )
1590
+ parsed_content = env.parse(chat_template)
1591
+ template_vars = jinja2.meta.find_undeclared_variables(parsed_content)
1592
+
1593
+ # We exclude chat_template from kwargs here, because
1594
+ # chat template has been already resolved at this stage
1595
+ unexpected_vars = {"chat_template"}
1596
+ accept_vars = (fn_kw | template_vars) - unexpected_vars
1597
+ return {
1598
+ k: v for k, v in chat_template_kwargs.items() if k in accept_vars
1599
+ }
1600
+
1601
+
1602
def apply_hf_chat_template(
1603
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1604
conversation: list[ConversationMessage],
@@ -1579,12 +1624,17 @@ def apply_hf_chat_template(
1624
)
1625
1626
try:
1627
+ resolved_kwargs = resolve_chat_template_kwargs(
1628
+ tokenizer=tokenizer,
1629
+ chat_template=hf_chat_template,
1630
+ chat_template_kwargs=kwargs,
1631
+ )
1632
return tokenizer.apply_chat_template(
1633
conversation=conversation, # type: ignore[arg-type]
1634
tools=tools, # type: ignore[arg-type]
1635
chat_template=hf_chat_template,
1636
tokenize=tokenize,
1587
- **kwargs,
1637
+ **resolved_kwargs,
1638
)
1639
1640
# External library exceptions can sometimes occur despite the framework's
vllm/entrypoints/openai/api_server.py
+1
@@ -1716,6 +1716,7 @@ async def init_app_state(
1716
request_logger=request_logger,
1717
chat_template=resolved_chat_template,
1718
chat_template_content_format=args.chat_template_content_format,
1719
+ trust_request_chat_template=args.trust_request_chat_template,
1720
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
1721
enable_auto_tools=args.enable_auto_tool_choice,
1722
exclude_tools_when_tool_choice_none=args.
vllm/entrypoints/openai/cli_args.py
+7
-3
@@ -103,9 +103,13 @@ class FrontendArgs:
103
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
104
"""The format to render message content within a chat template.
105
106
-* "string" will render the content as a string. Example: `"Hello World"`
107
-* "openai" will render the content as a list of dictionaries, similar to OpenAI
108
-schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
106
+ * "string" will render the content as a string. Example: `"Hello World"`
107
+ * "openai" will render the content as a list of dictionaries, similar to
108
+ OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
109
+ trust_request_chat_template: bool = False
110
+ """Whether to trust the chat template provided in the request. If False,
111
+ the server will always use the chat template specified by `--chat-template`
112
+ or the ones from tokenizer."""
113
response_role: str = "assistant"
114
"""The role name to return if `request.add_generation_prompt=true`."""
115
ssl_keyfile: Optional[str] = None
vllm/entrypoints/openai/serving_chat.py
+13
-1
@@ -68,6 +68,7 @@ class OpenAIServingChat(OpenAIServing):
68
request_logger: Optional[RequestLogger],
69
chat_template: Optional[str],
70
chat_template_content_format: ChatTemplateContentFormatOption,
71
+ trust_request_chat_template: bool = False,
72
return_tokens_as_token_ids: bool = False,
73
reasoning_parser: str = "",
74
enable_auto_tools: bool = False,
@@ -89,6 +90,7 @@ class OpenAIServingChat(OpenAIServing):
90
self.response_role = response_role
91
self.chat_template = chat_template
92
self.chat_template_content_format: Final = chat_template_content_format
93
+ self.trust_request_chat_template = trust_request_chat_template
94
self.enable_log_outputs = enable_log_outputs
95
96
# set up tool use
@@ -220,6 +222,16 @@ class OpenAIServingChat(OpenAIServing):
222
223
if not self.use_harmony:
224
# Common case.
225
+ request_chat_template = request.chat_template
226
+ chat_template_kwargs = request.chat_template_kwargs
227
+ if not self.trust_request_chat_template and (
228
+ request_chat_template is not None or
229
+ (chat_template_kwargs and
230
+ chat_template_kwargs.get("chat_template") is not None)):
231
+ return self.create_error_response(
232
+ "Chat template is passed with request, but "
233
+ "--trust-request-chat-template is not set. "
234
+ "Refused request with untrusted chat template.")
235
(
236
conversation,
237
request_prompts,
@@ -228,7 +240,7 @@ class OpenAIServingChat(OpenAIServing):
240
request,
241
tokenizer,
242
request.messages,
231
- chat_template=request.chat_template or self.chat_template,
243
+ chat_template=request_chat_template or self.chat_template,
244
chat_template_content_format=self.
245
chat_template_content_format,
246
add_generation_prompt=request.add_generation_prompt,