A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 19:00 UTC
[Frontend] [Bugfix] respect server-level default chat template kwargs in reasoning parser (#31581)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
cjackal committed
Jan 5, 2026 at 14:42 UTC
e2701cc52514c13a460e90746c901cce8ad753f9
2 files changed
+28
-5
vllm/entrypoints/openai/serving_chat.py
+12
-2
@@ -659,9 +659,14 @@ class OpenAIServingChat(OpenAIServing):
659
"Tokenizer not available when `skip_tokenizer_init=True`"
660
)
661
662
+ # Pass the same chat template kwargs as used in tokenization
663
+ chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
664
+ request.chat_template_kwargs,
665
+ self.default_chat_template_kwargs,
666
+ )
667
reasoning_parser = self.reasoning_parser(
668
tokenizer,
664
- chat_template_kwargs=request.chat_template_kwargs, # type: ignore
669
+ chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
670
)
671
except RuntimeError as e:
672
logger.exception("Error in reasoning parser creation.")
@@ -1437,9 +1442,14 @@ class OpenAIServingChat(OpenAIServing):
1442
"Tokenizer not available when `skip_tokenizer_init=True`"
1443
)
1444
1445
+ # Pass the same chat template kwargs as used in tokenization
1446
+ chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
1447
+ request.chat_template_kwargs,
1448
+ self.default_chat_template_kwargs,
1449
+ )
1450
reasoning_parser = self.reasoning_parser(
1451
tokenizer,
1442
- chat_template_kwargs=request.chat_template_kwargs, # type: ignore
1452
+ chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
1453
)
1454
except RuntimeError as e:
1455
logger.exception("Error in reasoning parser creation.")
vllm/entrypoints/openai/serving_engine.py
+16
-3
@@ -1146,6 +1146,18 @@ class OpenAIServing:
1146
)
1147
return None
1148
1149
+ @staticmethod
1150
+ def _prepare_extra_chat_template_kwargs(
1151
+ request_chat_template_kwargs: dict[str, Any] | None = None,
1152
+ default_chat_template_kwargs: dict[str, Any] | None = None,
1153
+ ) -> dict[str, Any]:
1154
+ """Helper to merge server-default and request-specific chat template kwargs."""
1155
+ request_chat_template_kwargs = request_chat_template_kwargs or {}
1156
+ if default_chat_template_kwargs is None:
1157
+ return request_chat_template_kwargs
1158
+ # Apply server defaults first, then request kwargs override.
1159
+ return default_chat_template_kwargs | request_chat_template_kwargs
1160
+
1161
async def _preprocess_chat(
1162
self,
1163
request: ChatLikeRequest | ResponsesRequest,
@@ -1184,9 +1196,10 @@ class OpenAIServing:
1196
tools=tool_dicts,
1197
documents=documents,
1198
)
1187
- if default_chat_template_kwargs:
1188
- _chat_template_kwargs.update(default_chat_template_kwargs)
1189
- _chat_template_kwargs.update(chat_template_kwargs or {})
1199
+ _chat_template_kwargs |= self._prepare_extra_chat_template_kwargs(
1200
+ chat_template_kwargs,
1201
+ default_chat_template_kwargs,
1202
+ )
1203
1204
request_prompt: str | list[int]
1205