[Bugfix][Frontend] Use a fresh parser per choice in non-streaming chat completions (#58939)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Sep 27, 2026 at 22:01 UTC
027b6f3a22f40b057474f053278e6323d81462b9
1 file changed
+22
-13
vllm/entrypoints/openai/chat_completion/serving.py
+22
-13
@@ -200,6 +200,20 @@ class OpenAIServingChat(GenerateBaseServing):
200
.chat_template_kwargs
201
)
202
203
+ def _make_parser(
204
+ self,
205
+ request: ChatCompletionRequest,
206
+ tokenizer: TokenizerLike,
207
+ chat_template_kwargs: dict[str, Any] | None,
208
+ ) -> Parser:
209
+ assert self.parser_cls is not None
210
+ return self.parser_cls(
211
+ tokenizer,
212
+ request.tools,
213
+ chat_template_kwargs=chat_template_kwargs,
214
+ model_config=self.model_config,
215
+ )
216
+
217
def _engine_chat_template_kwargs(
218
self, chat_template_kwargs: dict[str, Any]
219
) -> dict[str, Any]:
@@ -267,12 +281,7 @@ class OpenAIServingChat(GenerateBaseServing):
281
chat_template_kwargs = self._effective_chat_template_kwargs(request)
282
parser: Parser | None = None
283
if self.parser_cls is not None:
270
- parser = self.parser_cls(
271
- tokenizer,
272
- request.tools,
273
- chat_template_kwargs=chat_template_kwargs,
274
- model_config=self.model_config,
275
- )
284
+ parser = self._make_parser(request, tokenizer, chat_template_kwargs)
285
result = await self.render_chat_request(request)
286
if isinstance(result, ErrorResponse):
287
return result
@@ -490,12 +499,7 @@ class OpenAIServingChat(GenerateBaseServing):
499
"Tokenizer not available when `skip_tokenizer_init=True`"
500
)
501
parsers: list[Parser | None] = [
493
- self.parser_cls(
494
- tokenizer,
495
- request.tools,
496
- chat_template_kwargs=chat_template_kwargs,
497
- model_config=self.model_config,
498
- )
502
+ self._make_parser(request, tokenizer, chat_template_kwargs)
503
for _ in range(num_choices)
504
]
505
else:
@@ -961,10 +965,15 @@ class OpenAIServingChat(GenerateBaseServing):
965
tool_parser_cls = (
966
self.parser_cls.tool_parser_cls if self.parser_cls is not None else None
967
)
964
- for output in final_res.outputs:
968
+ for i, output in enumerate(final_res.outputs):
969
# check for error finish reason and raise GenerationError
970
# finish_reason='error' indicates a retryable request-level internal error
971
self._raise_if_error(output.finish_reason, request_id)
972
+ if i > 0 and parser is not None:
973
+ # Parsers are stateful: like streaming, use one per choice.
974
+ parser = self._make_parser(
975
+ request, tokenizer, self._effective_chat_template_kwargs(request)
976
+ )
977
token_ids = output.token_ids
978
out_logprobs = output.logprobs
979