@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2908094567
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 23:30 UTC

Add `/v1/chat/completions/batch` endpoint for batched chat completions (#38011)

Signed-off-by: Matej Rojec <64556640+MatejRojec@users.noreply.github.com>

Matej Rojec committed Mar 26, 2026 at 05:13 UTC 2908094567e7ebbf59b0f3b9be036d8929901971
8 files changed +771 -21
examples/online_serving/batched_chat_completions.py new
+194
@@ -0,0 +1,194 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Examples of batched chat completions via the vLLM OpenAI-compatible API.
4 +
5 +The /v1/chat/completions/batch endpoint accepts ``messages`` as a list of
6 +conversations. Each conversation is processed independently and the response
7 +contains one choice per conversation, indexed 0, 1, ..., N-1.
8 +
9 +Start a server first, e.g.:
10 + vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000
11 +
12 +Current limitations compared to /v1/chat/completions:
13 + - Streaming is not supported.
14 + - Tool use is not supported.
15 + - Beam search is not supported.
16 +"""
17 +
18 +import json
19 +import os
20 +
21 +import httpx
22 +
23 +BASE_URL = os.environ.get("VLLM_BASE_URL", "http://localhost:8000")
24 +MODEL = os.environ.get("VLLM_MODEL", "Qwen/Qwen2.5-1.5B-Instruct")
25 +BATCH_URL = f"{BASE_URL}/v1/chat/completions/batch"
26 +
27 +
28 +def post_batch(payload: dict) -> dict:
29 + response = httpx.post(BATCH_URL, json=payload, timeout=60)
30 + response.raise_for_status()
31 + return response.json()
32 +
33 +
34 +def main() -> None:
35 + print("=== Example 1a: single conversation (standard endpoint) ===")
36 + response = httpx.post(
37 + f"{BASE_URL}/v1/chat/completions",
38 + json={
39 + "model": MODEL,
40 + "messages": [{"role": "user", "content": "What is the capital of Japan?"}],
41 + },
42 + timeout=60,
43 + )
44 + response.raise_for_status()
45 + data = response.json()
46 + for choice in data["choices"]:
47 + print(f" [{choice['index']}] {choice['message']['content']}")
48 +
49 + print("\n=== Example 1b: batched plain text (2 conversations) ===")
50 + data = post_batch(
51 + {
52 + "model": MODEL,
53 + "messages": [
54 + [{"role": "user", "content": "What is the capital of France?"}],
55 + [{"role": "user", "content": "What is the capital of Japan?"}],
56 + ],
57 + }
58 + )
59 + for choice in data["choices"]:
60 + print(f" [{choice['index']}] {choice['message']['content']}")
61 +
62 + print("\n=== Example 2: batch with regex constraint (yes|no) ===")
63 + data = post_batch(
64 + {
65 + "model": MODEL,
66 + "messages": [
67 + [{"role": "user", "content": "Is the sky blue? Answer yes or no."}],
68 + [{"role": "user", "content": "Is fire cold? Answer yes or no."}],
69 + ],
70 + "structured_outputs": {"regex": "(yes|no)"},
71 + }
72 + )
73 + for choice in data["choices"]:
74 + print(f" [{choice['index']}] {choice['message']['content']}")
75 +
76 + print("\n=== Example 3: batch with json_schema ===")
77 + person_schema = {
78 + "type": "object",
79 + "properties": {
80 + "name": {"type": "string", "description": "Full name of the person"},
81 + "age": {"type": "integer", "description": "Age in years"},
82 + },
83 + "required": ["name", "age"],
84 + }
85 + data = post_batch(
86 + {
87 + "model": MODEL,
88 + "messages": [
89 + [
90 + {
91 + "role": "user",
92 + "content": "Describe the person: name Alice, age 30.",
93 + }
94 + ],
95 + [{"role": "user", "content": "Describe the person: name Bob, age 25."}],
96 + ],
97 + "response_format": {
98 + "type": "json_schema",
99 + "json_schema": {
100 + "name": "person",
101 + "strict": True,
102 + "schema": person_schema,
103 + },
104 + },
105 + }
106 + )
107 + for choice in data["choices"]:
108 + person = json.loads(choice["message"]["content"])
109 + print(f" [{choice['index']}] {person}")
110 +
111 + print("\n=== Example 4: batch book summaries ===")
112 + book_schema = {
113 + "type": "object",
114 + "properties": {
115 + "author": {
116 + "type": "string",
117 + "description": "Full name of the author",
118 + },
119 + "num_pages": {
120 + "type": "integer",
121 + "description": "Number of pages in the book",
122 + },
123 + "short_summary": {
124 + "type": "string",
125 + "description": "A one-sentence summary of the book",
126 + },
127 + "long_summary": {
128 + "type": "string",
129 + "description": (
130 + "A detailed two to three sentence summary covering "
131 + "the main themes and plot"
132 + ),
133 + },
134 + },
135 + "required": ["author", "num_pages", "short_summary", "long_summary"],
136 + }
137 + system_msg = {
138 + "role": "system",
139 + "content": (
140 + "You are a literary analyst. Extract structured information "
141 + "from book descriptions."
142 + ),
143 + }
144 + data = post_batch(
145 + {
146 + "model": MODEL,
147 + "messages": [
148 + [
149 + system_msg,
150 + {
151 + "role": "user",
152 + "content": (
153 + "Extract information from this book: '1984' by George"
154 + " Orwell, published in 1949, 328 pages. A dystopian"
155 + " novel set in a totalitarian society ruled by Big"
156 + " Brother, following Winston Smith as he secretly"
157 + " rebels against the oppressive Party that surveils"
158 + " and controls every aspect of life."
159 + ),
160 + },
161 + ],
162 + [
163 + system_msg,
164 + {
165 + "role": "user",
166 + "content": (
167 + "Extract information from this book: 'The Hitchhiker's"
168 + " Guide to the Galaxy' by Douglas Adams, published in"
169 + " 1979, 193 pages. A comedic science fiction novel"
170 + " following Arthur Dent, an ordinary Englishman who is"
171 + " whisked off Earth moments before it is demolished to"
172 + " make way for a hyperspace bypass, and his subsequent"
173 + " absurd adventures across the universe."
174 + ),
175 + },
176 + ],
177 + ],
178 + "response_format": {
179 + "type": "json_schema",
180 + "json_schema": {
181 + "name": "book_summary",
182 + "strict": True,
183 + "schema": book_schema,
184 + },
185 + },
186 + }
187 + )
188 + for choice in data["choices"]:
189 + book = json.loads(choice["message"]["content"])
190 + print(f" [{choice['index']}] {book}")
191 +
192 +
193 +if __name__ == "__main__":
194 + main()
tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py new
+113
@@ -0,0 +1,113 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import httpx
7 +import pytest
8 +
9 +from tests.utils import RemoteOpenAIServer
10 +
11 +# any model with a chat template defined in tokenizer_config should work here
12 +MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
13 +
14 +
15 +@pytest.fixture(scope="module")
16 +def default_server_args():
17 + return [
18 + # use half precision for speed and memory savings in CI environment
19 + "--max-model-len",
20 + "2048",
21 + "--max-num-seqs",
22 + "128",
23 + "--enforce-eager",
24 + ]
25 +
26 +
27 +@pytest.fixture(scope="module")
28 +def server(default_server_args):
29 + with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
30 + yield remote_server
31 +
32 +
33 +@pytest.mark.asyncio
34 +@pytest.mark.parametrize(
35 + "model_name",
36 + [MODEL_NAME],
37 +)
38 +async def test_batched_chat_completions(
39 + server: RemoteOpenAIServer, model_name: str
40 +) -> None:
41 + conversations = [
42 + [{"role": "user", "content": "Reply with exactly the word: alpha"}],
43 + [{"role": "user", "content": "Reply with exactly the word: beta"}],
44 + ]
45 +
46 + async with httpx.AsyncClient() as http_client:
47 + response = await http_client.post(
48 + f"{server.url_for('v1/chat/completions/batch')}",
49 + json={
50 + "model": model_name,
51 + "messages": conversations,
52 + },
53 + timeout=60,
54 + )
55 +
56 + assert response.status_code == 200, response.text
57 + data = response.json()
58 +
59 + choices = data["choices"]
60 + assert len(choices) == 2
61 +
62 + indices = {choice["index"] for choice in choices}
63 + assert indices == {0, 1}
64 +
65 + # Each conversation should produce a non-empty text response.
66 + for choice in choices:
67 + assert choice["message"]["content"]
68 +
69 +
70 +@pytest.mark.asyncio
71 +@pytest.mark.parametrize(
72 + "model_name",
73 + [MODEL_NAME],
74 +)
75 +async def test_batched_chat_completions_with_json_schema(
76 + server: RemoteOpenAIServer, model_name: str
77 +) -> None:
78 + schema = {
79 + "type": "object",
80 + "properties": {
81 + "answer": {"type": "string", "enum": ["yes", "no"]},
82 + },
83 + "required": ["answer"],
84 + }
85 + conversations = [
86 + [{"role": "user", "content": "Is the sky blue? Answer in JSON."}],
87 + [{"role": "user", "content": "Is fire cold? Answer in JSON."}],
88 + ]
89 +
90 + async with httpx.AsyncClient() as http_client:
91 + response = await http_client.post(
92 + f"{server.url_for('v1/chat/completions/batch')}",
93 + json={
94 + "model": model_name,
95 + "messages": conversations,
96 + "response_format": {
97 + "type": "json_schema",
98 + "json_schema": {"name": "answer", "schema": schema, "strict": True},
99 + },
100 + },
101 + timeout=60,
102 + )
103 +
104 + assert response.status_code == 200, response.text
105 + data = response.json()
106 +
107 + choices = data["choices"]
108 + assert len(choices) == 2
109 +
110 + for choice in choices:
111 + parsed = json.loads(choice["message"]["content"])
112 + assert "answer" in parsed
113 + assert parsed["answer"] in ("yes", "no")
tests/entrypoints/openai/test_openai_schema.py
+1
@@ -174,6 +174,7 @@ def test_openapi_stateless(case: Case):
174 timeout = {
175 # requires a longer timeout
176 ("POST", "/v1/chat/completions"): LONG_TIMEOUT_SECONDS,
177 + ("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS,
178 ("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS,
179 ("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS,
180 }.get(key, DEFAULT_TIMEOUT_SECONDS)
vllm/entrypoints/openai/chat_completion/api_router.py
+34
@@ -7,7 +7,9 @@ from http import HTTPStatus
7 from fastapi import APIRouter, Depends, FastAPI, Request
8 from fastapi.responses import JSONResponse, StreamingResponse
9
10 +from vllm.entrypoints.openai.chat_completion.batch_serving import OpenAIServingChatBatch
11 from vllm.entrypoints.openai.chat_completion.protocol import (
12 + BatchChatCompletionRequest,
13 ChatCompletionRequest,
14 ChatCompletionResponse,
15 )
@@ -31,6 +33,10 @@ def chat(request: Request) -> OpenAIServingChat | None:
33 return request.app.state.openai_serving_chat
34
35
36 +def batch_chat(request: Request) -> OpenAIServingChatBatch | None:
37 + return request.app.state.openai_serving_chat_batch
38 +
39 +
40 @router.post(
41 "/v1/chat/completions",
42 dependencies=[Depends(validate_json_request)],
@@ -68,5 +74,33 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
74 return StreamingResponse(content=generator, media_type="text/event-stream")
75
76
77 +@router.post(
78 + "/v1/chat/completions/batch",
79 + dependencies=[Depends(validate_json_request)],
80 + responses={
81 + HTTPStatus.OK.value: {},
82 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
83 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
84 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
85 + HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
86 + },
87 +)
88 +@with_cancellation
89 +@load_aware_call
90 +async def create_batch_chat_completion(
91 + request: BatchChatCompletionRequest, raw_request: Request
92 +):
93 + handler = batch_chat(raw_request)
94 + if handler is None:
95 + raise NotImplementedError("The model does not support Chat Completions API")
96 +
97 + result = await handler.create_batch_chat_completion(request, raw_request)
98 +
99 + if isinstance(result, ErrorResponse):
100 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
101 +
102 + return JSONResponse(content=result.model_dump())
103 +
104 +
105 def attach_router(app: FastAPI):
106 app.include_router(router)
vllm/entrypoints/openai/chat_completion/batch_serving.py new
+317
@@ -0,0 +1,317 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import asyncio
5 +import time
6 +from collections.abc import AsyncGenerator
7 +from http import HTTPStatus
8 +
9 +from fastapi import Request
10 +
11 +from vllm.entrypoints.chat_utils import ConversationMessage
12 +from vllm.entrypoints.openai.chat_completion.protocol import (
13 + BatchChatCompletionRequest,
14 + ChatCompletionResponse,
15 + ChatCompletionResponseChoice,
16 + ChatMessage,
17 +)
18 +from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
19 +from vllm.entrypoints.openai.engine.protocol import (
20 + ErrorResponse,
21 + RequestResponseMetadata,
22 + UsageInfo,
23 +)
24 +from vllm.entrypoints.utils import get_max_tokens
25 +from vllm.inputs import EngineInput
26 +from vllm.logger import init_logger
27 +from vllm.outputs import RequestOutput
28 +from vllm.reasoning import ReasoningParser
29 +from vllm.tokenizers import TokenizerLike
30 +from vllm.utils.async_utils import merge_async_iterators
31 +from vllm.utils.collection_utils import as_list
32 +
33 +logger = init_logger(__name__)
34 +
35 +
36 +class OpenAIServingChatBatch(OpenAIServingChat):
37 + """Extends OpenAIServingChat with the /v1/chat/completions/batch endpoint.
38 +
39 + Processes N conversations from a single request concurrently and returns
40 + one choice per conversation indexed 0, 1, ..., N-1.
41 + """
42 +
43 + async def render_batch_chat_request(
44 + self,
45 + request: BatchChatCompletionRequest,
46 + ) -> tuple[list[list[ConversationMessage]], list[EngineInput]] | ErrorResponse:
47 + """Validate the model and preprocess a batched chat completion request.
48 +
49 + Performs engine-aware checks then delegates per-conversation
50 + preprocessing to OpenAIServingRender, validating the chat template
51 + once for the whole batch.
52 +
53 + Returns:
54 + A tuple of (all_conversations, engine_prompts) on success — one
55 + entry per conversation — or an ErrorResponse on failure.
56 + """
57 + error_check_ret = await self._check_model(request)
58 + if error_check_ret is not None:
59 + logger.error("Error with model %s", error_check_ret)
60 + return error_check_ret
61 +
62 + if self.engine_client.errored:
63 + raise self.engine_client.dead_error
64 +
65 + render = self.openai_serving_render
66 +
67 + if not render.use_harmony:
68 + # Common case: validate the chat template once for the whole batch.
69 + error_check_ret = render.validate_chat_template(
70 + request_chat_template=request.chat_template,
71 + chat_template_kwargs=request.chat_template_kwargs,
72 + trust_request_chat_template=render.trust_request_chat_template,
73 + )
74 + if error_check_ret is not None:
75 + return error_check_ret
76 +
77 + tool_parser = render.tool_parser
78 + tool_dicts: list[dict] | None = None
79 +
80 + all_conversations: list[list[ConversationMessage]] = []
81 + all_engine_prompts: list[EngineInput] = []
82 +
83 + for messages in request.messages:
84 + single_request = request.to_chat_completion_request(messages)
85 + if render.use_harmony:
86 + conversation, engine_prompts = render._make_request_with_harmony(
87 + single_request, should_include_tools=tool_dicts is not None
88 + )
89 + else:
90 + conversation, engine_prompts = await render.preprocess_chat(
91 + single_request,
92 + messages,
93 + default_template=render.chat_template,
94 + default_template_content_format=render.chat_template_content_format,
95 + default_template_kwargs=render.default_chat_template_kwargs,
96 + tool_dicts=tool_dicts,
97 + tool_parser=tool_parser,
98 + )
99 + all_conversations.append(conversation)
100 + all_engine_prompts.append(engine_prompts[0])
101 +
102 + return all_conversations, all_engine_prompts
103 +
104 + async def create_batch_chat_completion(
105 + self,
106 + request: BatchChatCompletionRequest,
107 + raw_request: Request | None = None,
108 + ) -> ChatCompletionResponse | ErrorResponse:
109 + """Batch Chat Completion endpoint (/v1/chat/completions/batch).
110 +
111 + Processes N conversations from a single request concurrently and
112 + returns one choice per conversation indexed 0, 1, ..., N-1.
113 + Streaming, tool use, and beam search are not supported.
114 + """
115 + tokenizer = self.renderer.tokenizer
116 + assert tokenizer is not None
117 +
118 + reasoning_parser: ReasoningParser | None = None
119 + if self.reasoning_parser_cls:
120 + chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
121 + request.chat_template_kwargs,
122 + self.default_chat_template_kwargs,
123 + )
124 + reasoning_parser = self.reasoning_parser_cls(
125 + tokenizer,
126 + chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
127 + )
128 +
129 + render_result = await self.render_batch_chat_request(request)
130 + if isinstance(render_result, ErrorResponse):
131 + return render_result
132 + all_conversations, engine_prompts = render_result
133 +
134 + request_id = (
135 + f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
136 + )
137 + request_metadata = RequestResponseMetadata(request_id=request_id)
138 + if raw_request:
139 + raw_request.state.request_metadata = request_metadata
140 +
141 + lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
142 + model_name = self.models.model_name(lora_request)
143 + data_parallel_rank = self._get_data_parallel_rank(raw_request)
144 + max_model_len = self.model_config.max_model_len
145 +
146 + generators: list[AsyncGenerator[RequestOutput, None]] = []
147 + for i, engine_prompt in enumerate(engine_prompts):
148 + sub_request_id = f"{request_id}_{i}"
149 + max_tokens = get_max_tokens(
150 + max_model_len,
151 + request.max_completion_tokens
152 + if request.max_completion_tokens is not None
153 + else request.max_tokens,
154 + self._extract_prompt_len(engine_prompt),
155 + self.default_sampling_params,
156 + self.override_max_tokens,
157 + )
158 + single_request = request.to_chat_completion_request(request.messages[i])
159 + sampling_params = single_request.to_sampling_params(
160 + max_tokens, self.default_sampling_params
161 + )
162 + self._log_inputs(
163 + sub_request_id,
164 + engine_prompt,
165 + params=sampling_params,
166 + lora_request=lora_request,
167 + )
168 + trace_headers = (
169 + None
170 + if raw_request is None
171 + else await self._get_trace_headers(raw_request.headers)
172 + )
173 + generators.append(
174 + self.engine_client.generate(
175 + engine_prompt,
176 + sampling_params,
177 + sub_request_id,
178 + lora_request=lora_request,
179 + trace_headers=trace_headers,
180 + priority=request.priority if hasattr(request, "priority") else 0,
181 + data_parallel_rank=data_parallel_rank,
182 + reasoning_ended=None,
183 + )
184 + )
185 +
186 + return await self.chat_completion_full_generator_batch(
187 + request, # type: ignore[arg-type]
188 + generators,
189 + request_id,
190 + model_name,
191 + all_conversations,
192 + tokenizer,
193 + request_metadata,
194 + reasoning_parser,
195 + )
196 +
197 + async def chat_completion_full_generator_batch(
198 + self,
199 + request: BatchChatCompletionRequest, # type: ignore[override]
200 + generators: list[AsyncGenerator[RequestOutput, None]],
201 + request_id: str,
202 + model_name: str,
203 + all_conversations: list[list[ConversationMessage]],
204 + tokenizer: TokenizerLike,
205 + request_metadata: RequestResponseMetadata,
206 + reasoning_parser: ReasoningParser | None = None,
207 + ) -> ErrorResponse | ChatCompletionResponse:
208 + """Handle batched (non-streaming) chat completions.
209 +
210 + Fans out N generators (one per conversation in the batch), collects
211 + the final output for each, and assembles a single
212 + ``ChatCompletionResponse`` whose ``choices`` are indexed 0,...,N-1.
213 +
214 + Tool-use and streaming are rejected upstream by the
215 + ``check_batch_mode`` validator, so neither needs to be handled here.
216 + """
217 + created_time = int(time.time())
218 + role = self.get_chat_request_role(request) # type: ignore[arg-type]
219 +
220 + final_results: dict[int, RequestOutput] = {}
221 + try:
222 + async for prompt_idx, res in merge_async_iterators(*generators):
223 + final_results[prompt_idx] = res
224 + except asyncio.CancelledError:
225 + return self.create_error_response("Client disconnected")
226 +
227 + choices: list[ChatCompletionResponseChoice] = []
228 + total_prompt_tokens = 0
229 + total_completion_tokens = 0
230 +
231 + for prompt_idx in range(len(generators)):
232 + final_res = final_results.get(prompt_idx)
233 + if final_res is None:
234 + return self.create_error_response(
235 + f"No output received from the engine for prompt {prompt_idx}.",
236 + err_type="InternalServerError",
237 + status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
238 + )
239 +
240 + assert final_res.prompt_token_ids is not None
241 + num_prompt_tokens = len(final_res.prompt_token_ids)
242 + if final_res.encoder_prompt_token_ids is not None:
243 + num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
244 + total_prompt_tokens += num_prompt_tokens
245 + total_completion_tokens += sum(
246 + len(output.token_ids) for output in final_res.outputs
247 + )
248 +
249 + for output in final_res.outputs:
250 + self._raise_if_error(output.finish_reason, request_id)
251 +
252 + if request.logprobs and request.top_logprobs is not None:
253 + assert output.logprobs is not None, "Did not output logprobs"
254 + logprobs = self._create_chat_logprobs(
255 + token_ids=output.token_ids,
256 + top_logprobs=output.logprobs,
257 + num_output_top_logprobs=request.top_logprobs,
258 + tokenizer=tokenizer,
259 + return_as_token_id=request.return_token_ids,
260 + )
261 + else:
262 + logprobs = None
263 +
264 + if reasoning_parser:
265 + reasoning, content = reasoning_parser.extract_reasoning(
266 + output.text,
267 + request=request, # type: ignore[arg-type]
268 + )
269 + if not getattr(request, "include_reasoning", True):
270 + reasoning = None
271 + else:
272 + reasoning = None
273 + content = output.text
274 +
275 + message = ChatMessage(role=role, reasoning=reasoning, content=content)
276 +
277 + if request.echo:
278 + conversation = all_conversations[prompt_idx]
279 + last_msg_content: str | list[dict[str, str]] = ""
280 + if conversation and "content" in conversation[-1]:
281 + last_msg_content = conversation[-1]["content"] or ""
282 + if isinstance(last_msg_content, list):
283 + last_msg_content = "\n".join(
284 + msg["text"] for msg in last_msg_content
285 + )
286 + message.content = last_msg_content + (message.content or "")
287 +
288 + choice_data = ChatCompletionResponseChoice(
289 + index=prompt_idx,
290 + message=message,
291 + logprobs=logprobs,
292 + finish_reason=output.finish_reason
293 + if output.finish_reason
294 + else "stop",
295 + stop_reason=output.stop_reason,
296 + token_ids=(
297 + as_list(output.token_ids) if request.return_token_ids else None
298 + ),
299 + )
300 + choices.append(choice_data)
301 +
302 + usage = UsageInfo(
303 + prompt_tokens=total_prompt_tokens,
304 + completion_tokens=total_completion_tokens,
305 + total_tokens=total_prompt_tokens + total_completion_tokens,
306 + )
307 + request_metadata.final_usage_info = usage
308 +
309 + choices.sort(key=lambda c: c.index)
310 +
311 + return ChatCompletionResponse(
312 + id=request_id,
313 + created=created_time,
314 + model=model_name,
315 + choices=choices,
316 + usage=usage,
317 + )
vllm/entrypoints/openai/chat_completion/protocol.py
+80
@@ -787,3 +787,83 @@ class ChatCompletionRequest(OpenAIBaseModel):
787 if data.get("reasoning_effort") == "none":
788 data["include_reasoning"] = False
789 return data
790 +
791 +
792 +class BatchChatCompletionRequest(OpenAIBaseModel):
793 + """Request model for the /v1/chat/completions/batch endpoint.
794 +
795 + Accepts the same fields as ChatCompletionRequest except that ``messages``
796 + is a list of conversations (each conversation is a
797 + ``list[ChatCompletionMessageParam]``). Each conversation is processed
798 + independently and the response contains one choice per conversation,
799 + indexed 0, 1, ..., N-1.
800 +
801 + Current limitations compared to the single-conversation endpoint:
802 + - Streaming is not supported (``stream`` must be False or omitted).
803 + - Tool use is not supported (``tools`` must be omitted).
804 + - Beam search is not supported (``use_beam_search`` must be False or omitted).
805 + - The ``n`` parameter must be 1 (or omitted).
806 + """
807 +
808 + messages: list[list[ChatCompletionMessageParam]] = Field(..., min_length=1)
809 + model: str | None = None
810 +
811 + # Shared sampling / generation fields — mirror ChatCompletionRequest.
812 + frequency_penalty: float | None = 0.0
813 + logit_bias: dict[str, float] | None = None
814 + logprobs: bool | None = False
815 + top_logprobs: int | None = 0
816 + max_tokens: int | None = None
817 + max_completion_tokens: int | None = None
818 + n: int | None = 1
819 + presence_penalty: float | None = 0.0
820 + response_format: Any | None = None
821 + seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
822 + stop: str | list[str] | None = Field(default_factory=list)
823 + temperature: float | None = 0.7
824 + top_p: float | None = 1.0
825 + user: str | None = None
826 +
827 + # vLLM extensions
828 + best_of: int | None = None
829 + use_beam_search: bool = False
830 + top_k: int | None = None
831 + min_p: float | None = 0.0
832 + repetition_penalty: float | None = 1.0
833 + length_penalty: float | None = 1.0
834 + early_stopping: bool = False
835 + structured_outputs: StructuredOutputsParams | None = None
836 + request_id: str | None = None
837 + add_generation_prompt: bool = True
838 + continue_final_message: bool = False
839 + chat_template: str | None = None
840 + chat_template_kwargs: dict[str, Any] | None = None
841 + include_stop_str_in_output: bool = False
842 + guided_decoding_backend: str | None = None
843 + echo: bool = False
844 + return_token_ids: bool = False
845 +
846 + @model_validator(mode="before")
847 + @classmethod
848 + def check_batch_mode(cls, data: Any) -> Any:
849 + if isinstance(data, BatchChatCompletionRequest):
850 + data = data.model_dump(exclude_unset=True)
851 + if data.get("use_beam_search"):
852 + raise ValueError(
853 + "Batch chat completions do not support beam search. "
854 + "Please set `use_beam_search` to False."
855 + )
856 + n = data.get("n", 1)
857 + if n is not None and n != 1:
858 + raise ValueError(
859 + "Batch chat completions do not support `n > 1`. Please set `n` to 1."
860 + )
861 + return data
862 +
863 + def to_chat_completion_request(
864 + self, messages: list[ChatCompletionMessageParam]
865 + ) -> ChatCompletionRequest:
866 + """Build a single-conversation ChatCompletionRequest from one conversation."""
867 + data = self.model_dump(exclude={"messages"}, exclude_none=True)
868 + data["messages"] = messages
869 + return ChatCompletionRequest.model_validate(data)
vllm/entrypoints/openai/engine/serving.py
+5 -1
@@ -26,6 +26,7 @@ from vllm.entrypoints.chat_utils import (
26 )
27 from vllm.entrypoints.logger import RequestLogger
28 from vllm.entrypoints.openai.chat_completion.protocol import (
29 + BatchChatCompletionRequest,
30 ChatCompletionNamedToolChoiceParam,
31 ChatCompletionRequest,
32 ChatCompletionResponse,
@@ -124,7 +125,10 @@ CompletionLikeRequest: TypeAlias = (
125 )
126
127 ChatLikeRequest: TypeAlias = (
127 - ChatCompletionRequest | TokenizeChatRequest | PoolingChatRequest
128 + ChatCompletionRequest
129 + | BatchChatCompletionRequest
130 + | TokenizeChatRequest
131 + | PoolingChatRequest
132 )
133
134 SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
vllm/entrypoints/openai/generate/api_router.py
+27 -20
@@ -56,6 +56,9 @@ async def init_generate_state(
56 MCPToolServer,
57 ToolServer,
58 )
59 + from vllm.entrypoints.openai.chat_completion.batch_serving import (
60 + OpenAIServingChatBatch,
61 + )
62 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
63 from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
64 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
@@ -96,27 +99,31 @@ async def init_generate_state(
99 if "generate" in supported_tasks
100 else None
101 )
102 + _chat_kwargs = dict(
103 + engine_client=engine_client,
104 + models=state.openai_serving_models,
105 + response_role=args.response_role,
106 + openai_serving_render=state.openai_serving_render,
107 + request_logger=request_logger,
108 + chat_template=resolved_chat_template,
109 + chat_template_content_format=args.chat_template_content_format,
110 + default_chat_template_kwargs=args.default_chat_template_kwargs,
111 + trust_request_chat_template=args.trust_request_chat_template,
112 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
113 + enable_auto_tools=args.enable_auto_tool_choice,
114 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
115 + tool_parser=args.tool_call_parser,
116 + reasoning_parser=args.structured_outputs_config.reasoning_parser,
117 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
118 + enable_force_include_usage=args.enable_force_include_usage,
119 + enable_log_outputs=args.enable_log_outputs,
120 + enable_log_deltas=args.enable_log_deltas,
121 + )
122 state.openai_serving_chat = (
100 - OpenAIServingChat(
101 - engine_client,
102 - state.openai_serving_models,
103 - args.response_role,
104 - openai_serving_render=state.openai_serving_render,
105 - request_logger=request_logger,
106 - chat_template=resolved_chat_template,
107 - chat_template_content_format=args.chat_template_content_format,
108 - default_chat_template_kwargs=args.default_chat_template_kwargs,
109 - trust_request_chat_template=args.trust_request_chat_template,
110 - return_tokens_as_token_ids=args.return_tokens_as_token_ids,
111 - enable_auto_tools=args.enable_auto_tool_choice,
112 - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
113 - tool_parser=args.tool_call_parser,
114 - reasoning_parser=args.structured_outputs_config.reasoning_parser,
115 - enable_prompt_tokens_details=args.enable_prompt_tokens_details,
116 - enable_force_include_usage=args.enable_force_include_usage,
117 - enable_log_outputs=args.enable_log_outputs,
118 - enable_log_deltas=args.enable_log_deltas,
119 - )
123 + OpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None
124 + )
125 + state.openai_serving_chat_batch = (
126 + OpenAIServingChatBatch(**_chat_kwargs)
127 if "generate" in supported_tasks
128 else None
129 )