A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 23:45 UTC
[Frontend] Preserve structured output special tokens in offline LLM.chat (#39352)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
Luciano Martins committed
Apr 18, 2026 at 20:46 UTC
38907e4391c29465745f0a9e11dd7e3bae4b30a0
1 file changed
+58
vllm/entrypoints/llm.py
+58
@@ -1638,6 +1638,17 @@ class LLM:
1638
seq_params = self._params_to_seq(params, len(seq_convs))
1639
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
1640
1641
+ # When thinking is enabled or tools are provided, and the model
1642
+ # uses special tokens for structured output (e.g. Gemma4's
1643
+ # <|channel>, <|tool_call>, <|"|>), automatically set
1644
+ # skip_special_tokens=False so these tokens are preserved in
1645
+ # output.text for downstream parsing.
1646
+ needs_parsing = (
1647
+ chat_template_kwargs and chat_template_kwargs.get("enable_thinking")
1648
+ ) or tools
1649
+ if needs_parsing:
1650
+ self._adjust_params_for_parsing(seq_params)
1651
+
1652
return self._render_and_run_requests(
1653
prompts=(
1654
self._preprocess_chat_one(
@@ -1663,6 +1674,53 @@ class LLM:
1674
use_tqdm=use_tqdm,
1675
)
1676
1677
+ def _adjust_params_for_parsing(
1678
+ self, params: Sequence[SamplingParams | PoolingParams]
1679
+ ) -> None:
1680
+ """Set ``skip_special_tokens=False`` when the model encodes
1681
+ structured output syntax as special tokens.
1682
+
1683
+ Models like Gemma4 register thinking delimiters
1684
+ (``<|channel>``/``<channel|>``) and tool call tokens
1685
+ (``<|tool_call>``/``<tool_call|>``/``<|"|>``) as special tokens.
1686
+ The default ``skip_special_tokens=True`` strips them from
1687
+ ``output.text``, breaking parsing of both reasoning blocks and
1688
+ tool calls.
1689
+
1690
+ This is a no-op for models whose structured tokens are regular
1691
+ text tokens (e.g. DeepSeek's ``<think>``/``</think>``).
1692
+ """
1693
+ # The offline API currently lacks a unified rendering pipeline.
1694
+ # Until the planned Renderer refactor is complete, we hardcode
1695
+ # this token preservation logic specifically for Gemma4 models
1696
+ # to avoid regressions on other models.
1697
+ hf_config = getattr(self.model_config, "hf_config", None)
1698
+ architectures = getattr(hf_config, "architectures", [])
1699
+
1700
+ if any("Gemma4" in arch for arch in architectures):
1701
+ tokenizer = self.renderer.get_tokenizer()
1702
+ vocab = tokenizer.get_vocab()
1703
+ special_ids = set(getattr(tokenizer, "all_special_ids", []))
1704
+
1705
+ # Tokens used for thinking delimiters and tool call syntax
1706
+ # that some models (Gemma4) register as special tokens.
1707
+ structured_tokens = (
1708
+ "<|channel>",
1709
+ "<channel|>", # thinking delimiters
1710
+ "<|tool_call>",
1711
+ "<tool_call|>", # tool call delimiters
1712
+ '<|"|>', # string quoting in tool args
1713
+ )
1714
+ needs_special = any(
1715
+ vocab.get(tok) in special_ids
1716
+ for tok in structured_tokens
1717
+ if tok in vocab
1718
+ )
1719
+ if needs_special:
1720
+ for sp in params:
1721
+ if isinstance(sp, SamplingParams) and sp.skip_special_tokens:
1722
+ sp.skip_special_tokens = False
1723
+
1724
def _render_and_run_requests(
1725
self,
1726
prompts: Iterable[EngineInput],