@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4a718e770d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 18:15 UTC

[Bug] Fix Failure in /v1/chat/completions/render for Multimodal Requests (https://github.com/vllm-project/vllm/issues/35665) (#35684)

Sergey Zinchenko committed Mar 14, 2026 at 17:10 UTC 4a718e770d885f38e841d9dccebff3f777b3608d
13 files changed +559 -167
pyproject.toml
+1
@@ -167,6 +167,7 @@ fo = "fo"
167 nd = "nd"
168 eles = "eles"
169 datas = "datas"
170 +ser = "ser"
171 ure = "ure"
172
173 [tool.uv]
tests/entrypoints/openai/cpu/__init__.py
tests/entrypoints/openai/cpu/test_render.py
+96 -57
@@ -7,7 +7,7 @@ import httpx
7 import pytest
8 import pytest_asyncio
9
10 -from tests.utils import RemoteOpenAIServer
10 +from tests.utils import RemoteLaunchRenderServer
11
12 MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
13
@@ -16,7 +16,7 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
16 def server():
17 args: list[str] = []
18
19 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
19 + with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server:
20 yield remote_server
21
22
@@ -43,23 +43,20 @@ async def test_completion_render_basic(client):
43 assert response.status_code == 200
44 data = response.json()
45
46 - # Verify response structure
46 + # Verify response structure - list of GenerateRequest
47 assert isinstance(data, list)
48 assert len(data) > 0
49
50 - # Verify first prompt
50 + # Verify first prompt is a GenerateRequest
51 first_prompt = data[0]
52 - assert "prompt_token_ids" in first_prompt
53 - assert "prompt" in first_prompt
54 - assert isinstance(first_prompt["prompt_token_ids"], list)
55 - assert len(first_prompt["prompt_token_ids"]) > 0
56 - assert isinstance(first_prompt["prompt"], str)
57 -
58 - # Verify prompt text is preserved
59 - assert (
60 - "When should a chat-completions handler return an empty string?"
61 - in first_prompt["prompt"]
62 - )
52 + assert "token_ids" in first_prompt
53 + assert "sampling_params" in first_prompt
54 + assert "model" in first_prompt
55 + assert "request_id" in first_prompt
56 + assert isinstance(first_prompt["token_ids"], list)
57 + assert len(first_prompt["token_ids"]) > 0
58 + assert first_prompt["model"] == MODEL_NAME
59 + assert first_prompt["request_id"].startswith("cmpl-")
60
61
62 @pytest.mark.asyncio
@@ -84,36 +81,15 @@ async def test_chat_completion_render_basic(client):
81 assert response.status_code == 200
82 data = response.json()
83
87 - # Verify response structure - should be [conversation, engine_prompts]
88 - assert isinstance(data, list)
89 - assert len(data) == 2
90 -
91 - conversation, engine_prompts = data
92 -
93 - # Verify conversation
94 - assert isinstance(conversation, list)
95 - assert len(conversation) > 0
96 - assert conversation[0]["role"] == "user"
97 - assert "empty string" in conversation[0]["content"]
98 -
99 - # Verify engine_prompts
100 - assert isinstance(engine_prompts, list)
101 - assert len(engine_prompts) > 0
84 + # Verify response structure - should be a GenerateRequest
85 + assert isinstance(data, dict)
86 + assert "token_ids" in data
87 + assert isinstance(data["token_ids"], list)
88 + assert len(data["token_ids"]) > 0
89
103 - first_prompt = engine_prompts[0]
104 - assert "prompt_token_ids" in first_prompt
105 - assert "prompt" in first_prompt
106 - assert isinstance(first_prompt["prompt_token_ids"], list)
107 - assert len(first_prompt["prompt_token_ids"]) > 0
108 -
109 - # Verify chat template was applied (should have instruction markers)
110 - assert "[INST]" in first_prompt["prompt"]
111 - assert "[/INST]" in first_prompt["prompt"]
112 -
113 - # Verify token IDs are correctly preserved as integers
114 - token_ids = first_prompt["prompt_token_ids"]
90 + # Verify token IDs are integers and BOS token is present
91 + token_ids = data["token_ids"]
92 assert all(isinstance(tid, int) for tid in token_ids)
116 - # Verify BOS token (usually 1 for LLaMA models)
93 assert token_ids[0] == 1
94
95
@@ -131,15 +107,18 @@ async def test_completion_render_multiple_prompts(client):
107 assert response.status_code == 200
108 data = response.json()
109
134 - # Should return two prompts
110 + # Should return two GenerateRequest items
111 assert isinstance(data, list)
112 assert len(data) == 2
113
138 - # Verify both prompts have required fields
114 + # Verify both prompts have GenerateRequest fields
115 for prompt in data:
140 - assert "prompt_token_ids" in prompt
141 - assert "prompt" in prompt
142 - assert len(prompt["prompt_token_ids"]) > 0
116 + assert "token_ids" in prompt
117 + assert "sampling_params" in prompt
118 + assert "model" in prompt
119 + assert "request_id" in prompt
120 + assert len(prompt["token_ids"]) > 0
121 + assert prompt["request_id"].startswith("cmpl-")
122
123
124 @pytest.mark.asyncio
@@ -160,17 +139,49 @@ async def test_chat_completion_render_multi_turn(client):
139 assert response.status_code == 200
140 data = response.json()
141
163 - conversation, engine_prompts = data
142 + # Verify tokenization occurred
143 + assert isinstance(data, dict)
144 + assert "token_ids" in data
145 + assert isinstance(data["token_ids"], list)
146 + assert len(data["token_ids"]) > 0
147
165 - # Verify all messages preserved
166 - assert len(conversation) == 3
167 - assert conversation[0]["role"] == "user"
168 - assert conversation[1]["role"] == "assistant"
169 - assert conversation[2]["role"] == "user"
148
171 - # Verify tokenization occurred
172 - assert len(engine_prompts) > 0
173 - assert len(engine_prompts[0]["prompt_token_ids"]) > 0
149 +@pytest.mark.asyncio
150 +async def test_chat_completion_render_with_stream_true(client):
151 + """Render accepts stream params but still returns JSON (non-streamed)."""
152 +
153 + response = await client.post(
154 + "/v1/chat/completions/render",
155 + json={
156 + "model": MODEL_NAME,
157 + "stream": True,
158 + "stream_options": {
159 + "include_usage": True,
160 + "continuous_usage_stats": True,
161 + },
162 + "messages": [
163 + {
164 + "role": "user",
165 + "content": "Stream options should be accepted by /render.",
166 + }
167 + ],
168 + },
169 + )
170 +
171 + assert response.status_code == 200
172 + assert response.headers.get("content-type", "").startswith("application/json")
173 +
174 + data = response.json()
175 + assert isinstance(data, dict)
176 + assert "token_ids" in data
177 + assert isinstance(data["token_ids"], list)
178 + assert len(data["token_ids"]) > 0
179 +
180 + # /render should preserve stream fields on the returned token-in request.
181 + assert data.get("stream") is True
182 + assert isinstance(data.get("stream_options"), dict)
183 + assert data["stream_options"].get("include_usage") is True
184 + assert data["stream_options"].get("continuous_usage_stats") is True
185
186
187 @pytest.mark.asyncio
@@ -224,3 +235,31 @@ async def test_completion_render_no_generation(client):
235 assert response.status_code == 200
236 # Render should be fast (< 1 second) since no generation
237 assert elapsed < 1.0
238 +
239 +
240 +@pytest.mark.asyncio
241 +async def test_chat_completion_render_with_sampling_params(client):
242 + """Verify sampling params are correctly returned by /render."""
243 + response = await client.post(
244 + "/v1/chat/completions/render",
245 + json={
246 + "model": MODEL_NAME,
247 + "messages": [{"role": "user", "content": "Test sampling params"}],
248 + "temperature": 0.123,
249 + "top_p": 0.456,
250 + "frequency_penalty": 1.1,
251 + },
252 + )
253 +
254 + assert response.status_code == 200
255 + data = response.json()
256 +
257 + assert "sampling_params" in data
258 + sampling_params = data["sampling_params"]
259 +
260 + assert sampling_params.get("temperature") == 0.123
261 + assert sampling_params.get("top_p") == 0.456
262 + assert sampling_params.get("frequency_penalty") == 1.1
263 +
264 + # Check that internal fields are not present
265 + assert "_all_stop_token_ids" not in sampling_params
tests/entrypoints/openai/cpu/test_render_multimodal.py new
+155
@@ -0,0 +1,155 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""Multimodal tests for the /render endpoints that expose prompt preprocessing."""
5 +
6 +import httpx
7 +import pytest
8 +import pytest_asyncio
9 +
10 +from tests.utils import RemoteOpenAIServer
11 +from vllm.multimodal.utils import encode_image_url
12 +
13 +VISION_MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
14 +
15 +
16 +@pytest.fixture(scope="module")
17 +def vision_server():
18 + """Vision-capable server used for multimodal /render tests."""
19 +
20 + args = [
21 + "--enforce-eager",
22 + "--max-model-len",
23 + "100",
24 + "--max-num-seqs",
25 + "1",
26 + "--limit-mm-per-prompt.image",
27 + "1",
28 + "--limit-mm-per-prompt.video",
29 + "0",
30 + ]
31 +
32 + env_overrides: dict[str, str] = {}
33 +
34 + with RemoteOpenAIServer(
35 + VISION_MODEL_NAME,
36 + args,
37 + env_dict=env_overrides,
38 + ) as remote_server:
39 + yield remote_server
40 +
41 +
42 +@pytest_asyncio.fixture
43 +async def vision_client(vision_server):
44 + async with httpx.AsyncClient(
45 + base_url=vision_server.url_for(""), timeout=60.0
46 + ) as http_client:
47 + yield http_client
48 +
49 +
50 +@pytest.mark.asyncio
51 +async def test_chat_completion_render_with_base64_image_url(
52 + vision_client,
53 + local_asset_server,
54 +):
55 + """Render a multimodal chat request and verify tokens are returned."""
56 +
57 + image = local_asset_server.get_image_asset("RGBA_comp.png")
58 + data_url = encode_image_url(image, format="PNG")
59 +
60 + assert data_url.startswith("data:image/")
61 + assert ";base64," in data_url
62 +
63 + response = await vision_client.post(
64 + "/v1/chat/completions/render",
65 + json={
66 + "model": VISION_MODEL_NAME,
67 + "messages": [
68 + {
69 + "role": "user",
70 + "content": [
71 + {"type": "image_url", "image_url": {"url": data_url}},
72 + {"type": "text", "text": "What's in this image?"},
73 + ],
74 + }
75 + ],
76 + },
77 + )
78 +
79 + assert response.status_code == 200
80 +
81 + data = response.json()
82 + assert isinstance(data, dict)
83 + assert "token_ids" in data
84 + assert isinstance(data["token_ids"], list)
85 + assert len(data["token_ids"]) > 0
86 +
87 + # Verify multimodal features are populated
88 + assert "features" in data
89 + features = data["features"]
90 + assert features is not None
91 +
92 + # mm_hashes: should have an "image" key with a list of hash strings
93 + assert "mm_hashes" in features
94 + assert "image" in features["mm_hashes"]
95 + image_hashes = features["mm_hashes"]["image"]
96 + assert isinstance(image_hashes, list)
97 + assert len(image_hashes) > 0
98 + assert all(isinstance(h, str) for h in image_hashes)
99 +
100 + # mm_placeholders: should have an "image" key with offset/length dicts
101 + assert "mm_placeholders" in features
102 + assert "image" in features["mm_placeholders"]
103 + image_placeholders = features["mm_placeholders"]["image"]
104 + assert isinstance(image_placeholders, list)
105 + assert len(image_placeholders) > 0
106 + for p in image_placeholders:
107 + assert "offset" in p
108 + assert "length" in p
109 + assert isinstance(p["offset"], int)
110 + assert isinstance(p["length"], int)
111 + assert p["length"] > 0
112 +
113 +
114 +@pytest.mark.asyncio
115 +async def test_tokenize_matches_render_for_multimodal_input(
116 + vision_client,
117 + local_asset_server,
118 +):
119 + """`/tokenize` should match `/v1/chat/completions/render` token output."""
120 +
121 + image = local_asset_server.get_image_asset("RGBA_comp.png")
122 + data_url = encode_image_url(image, format="PNG")
123 +
124 + messages = [
125 + {
126 + "role": "user",
127 + "content": [
128 + {"type": "image_url", "image_url": {"url": data_url}},
129 + {"type": "text", "text": "What's in this image?"},
130 + ],
131 + }
132 + ]
133 +
134 + render_response = await vision_client.post(
135 + "/v1/chat/completions/render",
136 + json={
137 + "model": VISION_MODEL_NAME,
138 + "messages": messages,
139 + },
140 + )
141 + assert render_response.status_code == 200
142 + render_data = render_response.json()
143 +
144 + tokenize_response = await vision_client.post(
145 + "/tokenize",
146 + json={
147 + "model": VISION_MODEL_NAME,
148 + "messages": messages,
149 + },
150 + )
151 + assert tokenize_response.status_code == 200
152 + tokenize_data = tokenize_response.json()
153 +
154 + assert tokenize_data["tokens"] == render_data["token_ids"]
155 + assert tokenize_data["count"] == len(render_data["token_ids"])
tests/entrypoints/openai/test_launch_render.py
+24 -30
@@ -42,21 +42,12 @@ async def test_chat_render_basic(client):
42 assert response.status_code == 200
43 data = response.json()
44
45 - assert isinstance(data, list)
46 - assert len(data) == 2
47 -
48 - conversation, engine_prompts = data
49 -
50 - assert isinstance(conversation, list)
51 - assert conversation[0]["role"] == "user"
52 -
53 - assert isinstance(engine_prompts, list)
54 - assert len(engine_prompts) > 0
55 - first_prompt = engine_prompts[0]
56 - assert "prompt_token_ids" in first_prompt
57 - assert "prompt" in first_prompt
58 - assert isinstance(first_prompt["prompt_token_ids"], list)
59 - assert all(isinstance(t, int) for t in first_prompt["prompt_token_ids"])
45 + # Response should be a GenerateRequest dict
46 + assert isinstance(data, dict)
47 + assert "token_ids" in data
48 + assert isinstance(data["token_ids"], list)
49 + assert len(data["token_ids"]) > 0
50 + assert all(isinstance(t, int) for t in data["token_ids"])
51
52
53 @pytest.mark.asyncio
@@ -74,14 +65,12 @@ async def test_chat_render_multi_turn(client):
65 )
66
67 assert response.status_code == 200
77 - conversation, engine_prompts = response.json()
68 + data = response.json()
69
79 - assert len(conversation) == 3
80 - assert conversation[0]["role"] == "user"
81 - assert conversation[1]["role"] == "assistant"
82 - assert conversation[2]["role"] == "user"
83 - assert len(engine_prompts) > 0
84 - assert len(engine_prompts[0]["prompt_token_ids"]) > 0
70 + assert isinstance(data, dict)
71 + assert "token_ids" in data
72 + assert isinstance(data["token_ids"], list)
73 + assert len(data["token_ids"]) > 0
74
75
76 @pytest.mark.asyncio
@@ -118,11 +107,13 @@ async def test_completion_render_basic(client):
107 assert len(data) > 0
108
109 first_prompt = data[0]
121 - assert "prompt_token_ids" in first_prompt
122 - assert "prompt" in first_prompt
123 - assert isinstance(first_prompt["prompt_token_ids"], list)
124 - assert len(first_prompt["prompt_token_ids"]) > 0
125 - assert "Once upon a time" in first_prompt["prompt"]
110 + assert "token_ids" in first_prompt
111 + assert "sampling_params" in first_prompt
112 + assert "model" in first_prompt
113 + assert "request_id" in first_prompt
114 + assert isinstance(first_prompt["token_ids"], list)
115 + assert len(first_prompt["token_ids"]) > 0
116 + assert first_prompt["request_id"].startswith("cmpl-")
117
118
119 @pytest.mark.asyncio
@@ -142,9 +133,12 @@ async def test_completion_render_multiple_prompts(client):
133 assert len(data) == 2
134
135 for prompt in data:
145 - assert "prompt_token_ids" in prompt
146 - assert "prompt" in prompt
147 - assert len(prompt["prompt_token_ids"]) > 0
136 + assert "token_ids" in prompt
137 + assert "sampling_params" in prompt
138 + assert "model" in prompt
139 + assert "request_id" in prompt
140 + assert len(prompt["token_ids"]) > 0
141 + assert prompt["request_id"].startswith("cmpl-")
142
143
144 @pytest.mark.asyncio
vllm/entrypoints/openai/api_server.py
+4
@@ -368,6 +368,7 @@ async def init_app_state(
368 request_logger=request_logger,
369 chat_template=resolved_chat_template,
370 chat_template_content_format=args.chat_template_content_format,
371 + default_chat_template_kwargs=args.default_chat_template_kwargs,
372 trust_request_chat_template=args.trust_request_chat_template,
373 )
374
@@ -457,6 +458,9 @@ async def init_render_app_state(
458
459 state.openai_serving_models = model_registry
460
461 + # Expose tokenization via the render handler (no engine required).
462 + state.openai_serving_tokenization = state.openai_serving_render
463 +
464 state.vllm_config = vllm_config
465 # Disable stats logging — there is no engine to poll.
466 state.log_stats = False
vllm/entrypoints/openai/engine/protocol.py
-51
@@ -17,7 +17,6 @@ from pydantic import (
17
18 from vllm.entrypoints.chat_utils import make_tool_call_id
19 from vllm.logger import init_logger
20 -from vllm.sampling_params import SamplingParams
20 from vllm.utils import random_uuid
21 from vllm.utils.import_utils import resolve_obj_by_qualname
22
@@ -269,53 +268,3 @@ class GenerationError(Exception):
268 def __init__(self, message: str = "Internal server error"):
269 super().__init__(message)
270 self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
272 -
273 -
274 -####### Tokens IN <> Tokens OUT #######
275 -class GenerateRequest(BaseModel):
276 - request_id: str = Field(
277 - default_factory=random_uuid,
278 - description=(
279 - "The request_id related to this request. If the caller does "
280 - "not set it, a random_uuid will be generated. This id is used "
281 - "through out the inference process and return in response."
282 - ),
283 - )
284 - token_ids: list[int]
285 - """The token ids to generate text from."""
286 -
287 - # features: MultiModalFeatureSpec
288 - # TODO (NickLucche): implement once Renderer work is completed
289 - features: str | None = None
290 - """The processed MM inputs for the model."""
291 -
292 - sampling_params: SamplingParams
293 - """The sampling parameters for the model."""
294 -
295 - model: str | None = None
296 -
297 - stream: bool | None = False
298 - stream_options: StreamOptions | None = None
299 - cache_salt: str | None = Field(
300 - default=None,
301 - description=(
302 - "If specified, the prefix cache will be salted with the provided "
303 - "string to prevent an attacker to guess prompts in multi-user "
304 - "environments. The salt should be random, protected from "
305 - "access by 3rd parties, and long enough to be "
306 - "unpredictable (e.g., 43 characters base64-encoded, corresponding "
307 - "to 256 bit)."
308 - ),
309 - )
310 - priority: int = Field(
311 - default=0,
312 - description=(
313 - "The priority of the request (lower means earlier handling; "
314 - "default: 0). Any priority other than 0 will raise an error "
315 - "if the served model does not use priority scheduling."
316 - ),
317 - )
318 - kv_transfer_params: dict[str, Any] | None = Field(
319 - default=None,
320 - description="KVTransfer parameters used for disaggregated serving.",
321 - )
vllm/entrypoints/openai/server_utils.py
+3 -2
@@ -11,7 +11,7 @@ from contextlib import asynccontextmanager
11 from http import HTTPStatus
12
13 import pydantic
14 -from fastapi import FastAPI, HTTPException, Request, Response
14 +from fastapi import FastAPI, HTTPException, Request
15 from fastapi.exceptions import RequestValidationError
16 from fastapi.responses import JSONResponse
17 from starlette.concurrency import iterate_in_threadpool
@@ -350,7 +350,8 @@ async def engine_error_handler(
350 server=req.app.state.server,
351 engine=req.app.state.engine_client,
352 )
353 - return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
353 + err = create_error_response(exc)
354 + return JSONResponse(err.model_dump(), status_code=err.error.code)
355
356
357 async def exception_handler(req: Request, exc: Exception):
vllm/entrypoints/serve/disagg/protocol.py
+50 -10
@@ -2,20 +2,55 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from typing import Any
4
5 -from pydantic import BaseModel, Field
5 +from pydantic import BaseModel, Field, field_validator
6
7 from vllm.config import ModelConfig
8 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
9 -from vllm.entrypoints.openai.engine.protocol import (
10 - SamplingParams,
11 - StreamOptions,
12 -)
9 +from vllm.entrypoints.openai.engine.protocol import StreamOptions
10 from vllm.logprobs import Logprob
11 from vllm.renderers import TokenizeParams
12 +from vllm.sampling_params import SamplingParams
13 from vllm.utils import random_uuid
14
17 -
15 ####### Tokens IN <> Tokens OUT #######
16 +
17 +
18 +class PlaceholderRangeInfo(BaseModel):
19 + """Serializable placeholder location for a single multi-modal item."""
20 +
21 + offset: int
22 + """Start index of the placeholder tokens in the prompt."""
23 +
24 + length: int
25 + """Number of placeholder tokens."""
26 +
27 + # TODO: add ``is_embed: list[bool] | None`` once the /generate side
28 + # consumes features — some models (e.g. Qwen-VL) use sparse
29 + # placeholder masks that cannot be recomputed from offset+length alone.
30 +
31 +
32 +class MultiModalFeatures(BaseModel):
33 + """Lightweight multimodal metadata produced by the render step.
34 +
35 + Carries hashes (for cache lookup / identification) and placeholder
36 + positions so the downstream ``/generate`` service knows *where* in
37 + the token sequence each multimodal item lives.
38 +
39 + .. note:: Phase 1 — metadata only.
40 + Phase 2 should add ``mm_kwargs`` (processed tensor data) using a
41 + binary transport so the ``/generate`` side can skip re-processing.
42 + The ``/generate`` endpoint must also be updated to inject these
43 + features into ``ProcessorInputs`` before passing to
44 + ``InputProcessor.process_inputs``.
45 + """
46 +
47 + mm_hashes: dict[str, list[str]]
48 + """Per-modality item hashes, e.g. ``{"image": ["abc", "def"]}``."""
49 +
50 + mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
51 + """Per-modality placeholder ranges in the token sequence."""
52 +
53 +
54 class GenerateRequest(BaseModel):
55 request_id: str = Field(
56 default_factory=lambda: f"{random_uuid()}",
@@ -28,10 +63,15 @@ class GenerateRequest(BaseModel):
63 token_ids: list[int]
64 """The token ids to generate text from."""
65
31 - # features: MultiModalFeatureSpec
32 - # TODO (NickLucche): implement once Renderer work is completed
33 - features: str | None = None
34 - """The processed MM inputs for the model."""
66 + @field_validator("token_ids")
67 + @classmethod
68 + def validate_token_ids(cls, v: list[int]) -> list[int]:
69 + if any(t < 0 for t in v):
70 + raise ValueError("token_ids must not contain negative values")
71 + return v
72 +
73 + features: MultiModalFeatures | None = None
74 + """Multimodal hashes and placeholder positions (populated for MM inputs)."""
75
76 sampling_params: SamplingParams
77 """The sampling parameters for the model."""
vllm/entrypoints/serve/render/api_router.py
+5 -4
@@ -9,6 +9,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque
9 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
10 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11 from vllm.entrypoints.openai.utils import validate_json_request
12 +from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
13 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
14 from vllm.logger import init_logger
15
@@ -24,7 +25,7 @@ def render(request: Request) -> OpenAIServingRender | None:
25 @router.post(
26 "/v1/chat/completions/render",
27 dependencies=[Depends(validate_json_request)],
27 - response_model=list,
28 + response_model=GenerateRequest,
29 responses={
30 HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
31 HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
@@ -44,13 +45,13 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re
45 if isinstance(result, ErrorResponse):
46 return JSONResponse(content=result.model_dump(), status_code=result.error.code)
47
47 - return JSONResponse(content=result)
48 + return JSONResponse(content=result.model_dump())
49
50
51 @router.post(
52 "/v1/completions/render",
53 dependencies=[Depends(validate_json_request)],
53 - response_model=list,
54 + response_model=list[GenerateRequest],
55 responses={
56 HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
57 HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
@@ -67,7 +68,7 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
68 if isinstance(result, ErrorResponse):
69 return JSONResponse(content=result.model_dump(), status_code=result.error.code)
70
70 - return JSONResponse(content=result)
71 + return JSONResponse(content=[item.model_dump() for item in result])
72
73
74 def attach_router(app: FastAPI) -> None:
vllm/entrypoints/serve/render/serving.py
+146 -6
@@ -24,14 +24,29 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
24 parse_chat_inputs_to_harmony_messages,
25 render_for_completion,
26 )
27 -from vllm.entrypoints.utils import create_error_response
27 +from vllm.entrypoints.serve.disagg.protocol import (
28 + GenerateRequest,
29 + MultiModalFeatures,
30 + PlaceholderRangeInfo,
31 +)
32 +from vllm.entrypoints.utils import (
33 + create_error_response,
34 + get_max_tokens,
35 +)
36 from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
37 from vllm.logger import init_logger
38 +from vllm.multimodal.inputs import MultiModalHashes, MultiModalPlaceholderDict
39 from vllm.parser import ParserManager
40 from vllm.renderers import BaseRenderer, merge_kwargs
32 -from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
41 +from vllm.renderers.inputs.preprocess import (
42 + extract_prompt_components,
43 + extract_prompt_len,
44 + parse_model_prompt,
45 + prompt_to_seq,
46 +)
47 from vllm.tokenizers import TokenizerLike
48 from vllm.tool_parsers import ToolParser
49 +from vllm.utils import random_uuid
50 from vllm.utils.mistral import is_mistral_tokenizer
51 from vllm.utils.mistral import mt as _mt
52
@@ -83,10 +98,18 @@ class OpenAIServingRender:
98 self.supports_browsing = False
99 self.supports_code_interpreter = False
100
101 + self.default_sampling_params = model_config.get_diff_sampling_param()
102 + mc = model_config
103 + self.override_max_tokens = (
104 + self.default_sampling_params.get("max_tokens")
105 + if mc.generation_config not in ("auto", "vllm")
106 + else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
107 + )
108 +
109 async def render_chat_request(
110 self,
111 request: ChatCompletionRequest,
89 - ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
112 + ) -> GenerateRequest | ErrorResponse:
113 """Validate the model and preprocess a chat completion request.
114
115 This is the authoritative implementation used directly by the
@@ -96,7 +119,56 @@ class OpenAIServingRender:
119 if error_check_ret is not None:
120 logger.error("Error with model %s", error_check_ret)
121 return error_check_ret
99 - return await self.render_chat(request)
122 +
123 + if request.use_beam_search:
124 + return self.create_error_response(
125 + "Beam search is not supported by the render endpoint"
126 + )
127 +
128 + result = await self.render_chat(request)
129 + if isinstance(result, ErrorResponse):
130 + return result
131 +
132 + _, engine_prompts = result
133 +
134 + if len(engine_prompts) != 1:
135 + return self.create_error_response(
136 + f"Expected exactly 1 engine prompt, got {len(engine_prompts)}"
137 + )
138 +
139 + engine_prompt = engine_prompts[0]
140 +
141 + prompt_components = extract_prompt_components(self.model_config, engine_prompt)
142 + token_ids = prompt_components.token_ids
143 + if not token_ids:
144 + return self.create_error_response("No token_ids rendered")
145 + token_ids = list(token_ids)
146 +
147 + input_length = extract_prompt_len(self.model_config, engine_prompt)
148 + max_tokens = get_max_tokens(
149 + self.model_config.max_model_len,
150 + request.max_completion_tokens
151 + if request.max_completion_tokens is not None
152 + else request.max_tokens,
153 + input_length,
154 + self.default_sampling_params,
155 + self.override_max_tokens,
156 + )
157 + params = request.to_sampling_params(max_tokens, self.default_sampling_params)
158 +
159 + request_id = f"chatcmpl-{random_uuid()}"
160 +
161 + return GenerateRequest(
162 + request_id=request_id,
163 + token_ids=token_ids,
164 + features=self._extract_mm_features(engine_prompt),
165 + sampling_params=params,
166 + model=request.model,
167 + stream=bool(request.stream),
168 + stream_options=(request.stream_options if request.stream else None),
169 + cache_salt=request.cache_salt,
170 + priority=request.priority,
171 + )
172
173 async def render_chat(
174 self,
@@ -183,7 +255,7 @@ class OpenAIServingRender:
255 async def render_completion_request(
256 self,
257 request: CompletionRequest,
186 - ) -> list[ProcessorInputs] | ErrorResponse:
258 + ) -> list[GenerateRequest] | ErrorResponse:
259 """Validate the model and preprocess a completion request.
260
261 This is the authoritative implementation used directly by the
@@ -192,7 +264,48 @@ class OpenAIServingRender:
264 error_check_ret = await self._check_model(request)
265 if error_check_ret is not None:
266 return error_check_ret
195 - return await self.render_completion(request)
267 + result = await self.render_completion(request)
268 + if isinstance(result, ErrorResponse):
269 + return result
270 + generate_requests: list[GenerateRequest] = []
271 + for engine_prompt in result:
272 + prompt_components = extract_prompt_components(
273 + self.model_config, engine_prompt
274 + )
275 + token_ids = prompt_components.token_ids
276 + if not token_ids:
277 + return self.create_error_response("No token_ids rendered")
278 + token_ids = list(token_ids)
279 +
280 + input_length = extract_prompt_len(self.model_config, engine_prompt)
281 + max_tokens = get_max_tokens(
282 + self.model_config.max_model_len,
283 + request.max_tokens,
284 + input_length,
285 + self.default_sampling_params,
286 + self.override_max_tokens,
287 + )
288 + params = request.to_sampling_params(
289 + max_tokens, self.default_sampling_params
290 + )
291 +
292 + request_id = f"cmpl-{random_uuid()}"
293 +
294 + generate_requests.append(
295 + GenerateRequest(
296 + request_id=request_id,
297 + token_ids=token_ids,
298 + features=self._extract_mm_features(engine_prompt),
299 + sampling_params=params,
300 + model=request.model,
301 + stream=bool(request.stream),
302 + stream_options=(request.stream_options if request.stream else None),
303 + cache_salt=request.cache_salt,
304 + priority=request.priority,
305 + )
306 + )
307 +
308 + return generate_requests
309
310 async def render_completion(
311 self,
@@ -223,6 +336,33 @@ class OpenAIServingRender:
336
337 return engine_prompts
338
339 + @staticmethod
340 + def _extract_mm_features(
341 + engine_prompt: ProcessorInputs,
342 + ) -> MultiModalFeatures | None:
343 + """Extract multimodal metadata from a rendered engine prompt.
344 +
345 + Returns ``None`` for text-only prompts.
346 + """
347 + if engine_prompt.get("type") != "multimodal":
348 + return None
349 +
350 + # At this point engine_prompt is a MultiModalInputs TypedDict.
351 + mm_hashes: MultiModalHashes = engine_prompt["mm_hashes"] # type: ignore[typeddict-item]
352 + raw_placeholders: MultiModalPlaceholderDict = engine_prompt["mm_placeholders"] # type: ignore[typeddict-item]
353 +
354 + mm_placeholders = {
355 + modality: [
356 + PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
357 + ]
358 + for modality, ranges in raw_placeholders.items()
359 + }
360 +
361 + return MultiModalFeatures(
362 + mm_hashes=mm_hashes,
363 + mm_placeholders=mm_placeholders,
364 + )
365 +
366 def _make_request_with_harmony(
367 self,
368 request: ChatCompletionRequest,
vllm/entrypoints/serve/tokenize/serving.py
+6 -3
@@ -35,6 +35,7 @@ class OpenAIServingTokenization(OpenAIServing):
35 request_logger: RequestLogger | None,
36 chat_template: str | None,
37 chat_template_content_format: ChatTemplateContentFormatOption,
38 + default_chat_template_kwargs: dict[str, Any] | None = None,
39 trust_request_chat_template: bool = False,
40 ) -> None:
41 super().__init__(
@@ -45,6 +46,7 @@ class OpenAIServingTokenization(OpenAIServing):
46
47 self.chat_template = chat_template
48 self.chat_template_content_format: Final = chat_template_content_format
49 + self.default_chat_template_kwargs = default_chat_template_kwargs or {}
50 self.trust_request_chat_template = trust_request_chat_template
51
52 async def create_tokenize(
@@ -79,7 +81,7 @@ class OpenAIServingTokenization(OpenAIServing):
81 request.messages,
82 default_template=self.chat_template,
83 default_template_content_format=self.chat_template_content_format,
82 - default_template_kwargs=None,
84 + default_template_kwargs=self.default_chat_template_kwargs,
85 tool_dicts=tool_dicts,
86 )
87 else:
@@ -98,8 +100,9 @@ class OpenAIServingTokenization(OpenAIServing):
100 lora_request=lora_request,
101 )
102
101 - if "prompt_token_ids" in engine_prompt:
102 - input_ids.extend(engine_prompt["prompt_token_ids"]) # type: ignore[typeddict-item]
103 + prompt_components = self._extract_prompt_components(engine_prompt)
104 + if prompt_components.token_ids is not None:
105 + input_ids.extend(prompt_components.token_ids)
106
107 token_strs = None
108 if request.return_token_strs:
vllm/v1/serial_utils.py
+69 -4
@@ -8,7 +8,7 @@ from collections.abc import Callable, Sequence
8 from functools import partial
9 from inspect import isclass
10 from types import FunctionType
11 -from typing import Any, TypeAlias, get_type_hints
11 +from typing import Any, ClassVar, TypeAlias, cast, get_type_hints
12
13 import cloudpickle
14 import msgspec
@@ -460,6 +460,19 @@ def run_method(
460
461
462 class PydanticMsgspecMixin:
463 + """Make a ``msgspec.Struct`` compatible with Pydantic for both
464 + **validation** (JSON/dict -> Struct) and **serialization**
465 + (Struct -> JSON-safe dict).
466 +
467 + Subclasses may set ``__pydantic_msgspec_exclude__`` (a ``set[str]``)
468 + to list non-underscore field names that should also be stripped from
469 + serialized output. Fields whose names start with ``_`` are always
470 + excluded automatically.
471 + """
472 +
473 + # Subclasses can override to exclude additional public-but-internal keys.
474 + __pydantic_msgspec_exclude__: ClassVar[set[str]] = set()
475 +
476 @classmethod
477 def __get_pydantic_core_schema__(
478 cls, source_type: Any, handler: GetCoreSchemaHandler
@@ -476,32 +489,62 @@ class PydanticMsgspecMixin:
489 # Build the Pydantic typed_dict_field for each msgspec field
490 fields = {}
491 for name, hint in type_hints.items():
492 + if name not in msgspec_fields:
493 + # Skip ClassVar and other non-struct annotations.
494 + continue
495 + # Skip private fields — they are excluded from serialization
496 + # and should not appear in the generated JSON/OpenAPI schema.
497 + if name.startswith("_"):
498 + continue
499 msgspec_field = msgspec_fields[name]
500
501 # typed_dict_field using the handler to get the schema
502 field_schema = handler(hint)
503
504 # Add default value to the schema.
505 + # Mark fields with defaults as not required so the generated
506 + # JSON Schema stays consistent with ``omit_defaults=True``
507 + # serialization (fields at their default value may be absent).
508 if msgspec_field.default_factory is not msgspec.NODEFAULT:
509 wrapped_schema = core_schema.with_default_schema(
510 schema=field_schema,
511 default_factory=msgspec_field.default_factory,
512 )
490 - fields[name] = core_schema.typed_dict_field(wrapped_schema)
513 + fields[name] = core_schema.typed_dict_field(
514 + wrapped_schema, required=False
515 + )
516 elif msgspec_field.default is not msgspec.NODEFAULT:
517 wrapped_schema = core_schema.with_default_schema(
518 schema=field_schema,
519 default=msgspec_field.default,
520 )
496 - fields[name] = core_schema.typed_dict_field(wrapped_schema)
521 + fields[name] = core_schema.typed_dict_field(
522 + wrapped_schema, required=False
523 + )
524 else:
525 # No default, so Pydantic will treat it as required
526 fields[name] = core_schema.typed_dict_field(field_schema)
500 - return core_schema.no_info_after_validator_function(
527 + typed_dict_then_convert = core_schema.no_info_after_validator_function(
528 cls._validate_msgspec,
529 core_schema.typed_dict_schema(fields),
530 )
531
532 + # Build a serializer that strips private / excluded fields.
533 + serializer = core_schema.plain_serializer_function_ser_schema(
534 + cls._serialize_msgspec,
535 + info_arg=False,
536 + )
537 +
538 + # Accept either an already-constructed msgspec.Struct instance or a
539 + # JSON/dict-like payload.
540 + return core_schema.union_schema(
541 + [
542 + core_schema.is_instance_schema(source_type),
543 + typed_dict_then_convert,
544 + ],
545 + serialization=serializer,
546 + )
547 +
548 @classmethod
549 def _validate_msgspec(cls, value: Any) -> Any:
550 """Validate and convert input to msgspec.Struct instance."""
@@ -510,3 +553,25 @@ class PydanticMsgspecMixin:
553 if isinstance(value, dict):
554 return cls(**value)
555 return msgspec.convert(value, type=cls)
556 +
557 + @staticmethod
558 + def _serialize_msgspec(value: Any) -> Any:
559 + """Serialize a msgspec.Struct to a JSON-compatible dict, stripping
560 + private (``_``-prefixed) and explicitly excluded fields.
561 +
562 + Uses ``msgspec.to_builtins`` which respects ``omit_defaults=True``,
563 + so only fields that differ from their declared defaults are included.
564 + """
565 + raw = msgspec.to_builtins(value)
566 + if not isinstance(raw, dict):
567 + return raw
568 +
569 + exclude: set[str] = cast(
570 + set[str],
571 + getattr(type(value), "__pydantic_msgspec_exclude__", set()),
572 + )
573 + for key in list(raw):
574 + if key.startswith("_") or key in exclude:
575 + del raw[key]
576 +
577 + return raw