[Bugfix][Frontend] Sample batched chat completions from the adjusted requests (#58929)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Sep 27, 2026 at 21:46 UTC
4d07e9065357fa7c9759aa374d9d2fb00fb9f1fd
1 file changed
+20
-13
vllm/entrypoints/openai/chat_completion/batch_serving.py
+20
-13
@@ -12,6 +12,7 @@ from vllm.entrypoints.chat_utils import ConversationMessage
12
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
13
from vllm.entrypoints.openai.chat_completion.protocol import (
14
BatchChatCompletionRequest,
15
+ ChatCompletionRequest,
16
ChatCompletionResponse,
17
ChatCompletionResponseChoice,
18
ChatMessage,
@@ -40,7 +41,14 @@ class OpenAIServingChatBatch(OpenAIServingChat):
41
async def render_batch_chat_request(
42
self,
43
request: BatchChatCompletionRequest,
43
- ) -> tuple[list[list[ConversationMessage]], list[EngineInput]] | ErrorResponse:
44
+ ) -> (
45
+ tuple[
46
+ list[ChatCompletionRequest],
47
+ list[list[ConversationMessage]],
48
+ list[EngineInput],
49
+ ]
50
+ | ErrorResponse
51
+ ):
52
"""Validate the model and preprocess a batched chat completion request.
53
54
Performs engine-aware checks then delegates per-conversation
@@ -48,8 +56,10 @@ class OpenAIServingChatBatch(OpenAIServingChat):
56
once for the whole batch.
57
58
Returns:
51
- A tuple of (all_conversations, engine_prompts) on success — one
52
- entry per conversation — or an ErrorResponse on failure.
59
+ A tuple of (single_requests, all_conversations, engine_prompts) on
60
+ success — one entry per conversation — or an ErrorResponse on
61
+ failure. The renderer adjusts ``single_requests`` in place, so
62
+ sampling must read them rather than rebuilding from ``request``.
63
64
"""
65
error_check_ret = await self._check_model(request)
@@ -74,6 +84,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
84
parser = renderer.parser
85
tool_dicts: list[dict] | None = None
86
87
+ single_requests: list[ChatCompletionRequest] = []
88
all_conversations: list[list[ConversationMessage]] = []
89
all_engine_prompts: list[EngineInput] = []
90
@@ -93,10 +104,11 @@ class OpenAIServingChatBatch(OpenAIServingChat):
104
tool_dicts=tool_dicts,
105
parser=parser,
106
)
107
+ single_requests.append(single_request)
108
all_conversations.append(conversation)
109
all_engine_prompts.append(engine_prompts[0])
110
99
- return all_conversations, all_engine_prompts
111
+ return single_requests, all_conversations, all_engine_prompts
112
113
async def create_batch_chat_completion(
114
self,
@@ -111,10 +123,10 @@ class OpenAIServingChatBatch(OpenAIServingChat):
123
"""
124
tokenizer = self.renderer.tokenizer
125
assert tokenizer is not None
114
- single_requests = [
115
- request.to_chat_completion_request(messages)
116
- for messages in request.messages
117
- ]
126
+ render_result = await self.render_batch_chat_request(request)
127
+ if isinstance(render_result, ErrorResponse):
128
+ return render_result
129
+ single_requests, all_conversations, engine_prompts = render_result
130
131
parser: Parser | None = None
132
if self.parser_cls is not None:
@@ -127,11 +139,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
139
chat_template_kwargs=chat_template_kwargs,
140
)
141
130
- render_result = await self.render_batch_chat_request(request)
131
- if isinstance(render_result, ErrorResponse):
132
- return render_result
133
- all_conversations, engine_prompts = render_result
134
-
142
request_id = (
143
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
144
)