A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Enforce tokenize=False when applying chat template (#27205)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Russell Bryant committed
Oct 20, 2025 at 22:57 UTC
3ada34f9cb4d1af763fdfa3b481862a93eb6bd2b
2 files changed
+31
-12
tests/entrypoints/test_chat_utils.py
+12
@@ -1810,6 +1810,7 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
1810
"unsed_kwargs_2": "abc",
1811
# should not appear
1812
"chat_template": "{% Hello world! %}",
1813
+ "tokenize": True,
1814
# used by tokenizer
1815
"continue_final_message": True,
1816
"tools": tools,
@@ -1846,10 +1847,21 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
1847
tools=tools,
1848
model_config=model_config,
1849
)
1850
+ with pytest.raises(
1851
+ ValueError, match="Found unexpected chat template kwargs from request"
1852
+ ):
1853
+ # should raise error if `chat_template_kwargs` contains
1854
+ # `chat_template` or `tokenize`
1855
+ resolve_chat_template_kwargs(
1856
+ tokenizer,
1857
+ chat_template=chat_template,
1858
+ chat_template_kwargs=chat_template_kwargs,
1859
+ )
1860
resolved_chat_template_kwargs = resolve_chat_template_kwargs(
1861
tokenizer,
1862
chat_template=chat_template,
1863
chat_template_kwargs=chat_template_kwargs,
1864
+ raise_on_unexpected=False,
1865
)
1866
assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs
1867
vllm/entrypoints/chat_utils.py
+19
-12
@@ -1499,18 +1499,25 @@ def resolve_chat_template_kwargs(
1499
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
1500
chat_template: str,
1501
chat_template_kwargs: dict[str, Any],
1502
+ raise_on_unexpected: bool = True,
1503
) -> dict[str, Any]:
1504
+ # We exclude chat_template from kwargs here, because
1505
+ # chat template has been already resolved at this stage
1506
+ unexpected_vars = {"chat_template", "tokenize"}
1507
+ if raise_on_unexpected and (
1508
+ unexpected_in_kwargs := unexpected_vars & chat_template_kwargs.keys()
1509
+ ):
1510
+ raise ValueError(
1511
+ "Found unexpected chat template kwargs from request: "
1512
+ f"{unexpected_in_kwargs}"
1513
+ )
1514
+
1515
fn_kw = {
1516
k
1517
for k in chat_template_kwargs
1518
if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False)
1519
}
1508
-
1520
template_vars = _cached_resolve_chat_template_kwargs(chat_template)
1510
-
1511
- # We exclude chat_template from kwargs here, because
1512
- # chat template has been already resolved at this stage
1513
- unexpected_vars = {"chat_template"}
1521
accept_vars = (fn_kw | template_vars) - unexpected_vars
1522
return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars}
1523
@@ -1522,7 +1529,6 @@ def apply_hf_chat_template(
1529
tools: list[dict[str, Any]] | None,
1530
*,
1531
model_config: ModelConfig,
1525
- tokenize: bool = False, # Different from HF's default
1532
**kwargs: Any,
1533
) -> str:
1534
hf_chat_template = resolve_hf_chat_template(
@@ -1539,17 +1545,18 @@ def apply_hf_chat_template(
1545
"does not define one."
1546
)
1547
1548
+ resolved_kwargs = resolve_chat_template_kwargs(
1549
+ tokenizer=tokenizer,
1550
+ chat_template=hf_chat_template,
1551
+ chat_template_kwargs=kwargs,
1552
+ )
1553
+
1554
try:
1543
- resolved_kwargs = resolve_chat_template_kwargs(
1544
- tokenizer=tokenizer,
1545
- chat_template=hf_chat_template,
1546
- chat_template_kwargs=kwargs,
1547
- )
1555
return tokenizer.apply_chat_template(
1556
conversation=conversation, # type: ignore[arg-type]
1557
tools=tools, # type: ignore[arg-type]
1558
chat_template=hf_chat_template,
1552
- tokenize=tokenize,
1559
+ tokenize=False,
1560
**resolved_kwargs,
1561
)
1562