@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 43f0e024bc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Render] Add `/derender` endpoints for disaggregated postprocessing (#43606)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Martin Hickey committed Jun 13, 2026 at 06:55 UTC 43f0e024bcc304e88b5be47555daabf795582354
7 files changed +898 -9
tests/entrypoints/serve/render/test_derender.py new
+488
@@ -0,0 +1,488 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
5 +
6 +import httpx
7 +import pytest
8 +import pytest_asyncio
9 +
10 +from tests.utils import RemoteLaunchRenderServer
11 +
12 +MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
13 +
14 +
15 +@pytest.fixture(scope="module")
16 +def server():
17 + with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server:
18 + yield remote_server
19 +
20 +
21 +@pytest_asyncio.fixture
22 +async def client(server):
23 + async with httpx.AsyncClient(
24 + base_url=server.url_for(""), timeout=30.0
25 + ) as http_client:
26 + yield http_client
27 +
28 +
29 +# ---------------------------------------------------------------------------
30 +# Helpers
31 +# ---------------------------------------------------------------------------
32 +
33 +
34 +async def _render_chat(client: httpx.AsyncClient) -> dict:
35 + """Render a minimal chat request and return the GenerateRequest dict."""
36 + resp = await client.post(
37 + "/v1/chat/completions/render",
38 + json={
39 + "model": MODEL_NAME,
40 + "messages": [{"role": "user", "content": "Hello"}],
41 + },
42 + )
43 + assert resp.status_code == 200
44 + return resp.json()
45 +
46 +
47 +def _make_generate_response(
48 + token_ids: list[int] | None,
49 + request_id: str = "chatcmpl-test-id",
50 + finish_reason: str = "stop",
51 + logprobs: dict | None = None,
52 + prompt_logprobs: list | None = None,
53 + kv_transfer_params: dict | None = None,
54 +) -> dict:
55 + choice: dict = {
56 + "index": 0,
57 + "token_ids": token_ids,
58 + "finish_reason": finish_reason,
59 + "logprobs": logprobs,
60 + }
61 + return {
62 + "request_id": request_id,
63 + "choices": [choice],
64 + "prompt_logprobs": prompt_logprobs,
65 + "kv_transfer_params": kv_transfer_params,
66 + }
67 +
68 +
69 +def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict:
70 + entry = {
71 + "token": f"token_id:{token_id}",
72 + "logprob": -1.0,
73 + "bytes": None,
74 + "top_logprobs": [
75 + {"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None}
76 + ],
77 + }
78 + return {"content": [entry]}
79 +
80 +
81 +# ---------------------------------------------------------------------------
82 +# Chat derender tests
83 +# ---------------------------------------------------------------------------
84 +
85 +
86 +@pytest.mark.asyncio
87 +async def test_derender_chat_roundtrip(client):
88 + """Render then derender: decoded content should be a non-empty string."""
89 + gen_req = await _render_chat(client)
90 + # Use the first 5 rendered token IDs as synthetic "generated" tokens.
91 + synthetic_ids = gen_req["token_ids"][:5]
92 +
93 + response = await client.post(
94 + "/v1/chat/completions/derender",
95 + json={
96 + "model": MODEL_NAME,
97 + "generate_response": _make_generate_response(synthetic_ids),
98 + },
99 + )
100 + assert response.status_code == 200
101 + data = response.json()
102 + assert data["object"] == "chat.completion"
103 + assert len(data["choices"]) == 1
104 + assert data["choices"][0]["message"]["content"]
105 + assert data["choices"][0]["message"]["role"] == "assistant"
106 +
107 +
108 +@pytest.mark.asyncio
109 +async def test_derender_chat_usage(client):
110 + """Supplied prompt_tokens flows through into usage correctly."""
111 + gen_req = await _render_chat(client)
112 + synthetic_ids = gen_req["token_ids"][:3]
113 +
114 + response = await client.post(
115 + "/v1/chat/completions/derender",
116 + json={
117 + "model": MODEL_NAME,
118 + "generate_response": _make_generate_response(synthetic_ids),
119 + "prompt_tokens": 10,
120 + },
121 + )
122 + assert response.status_code == 200
123 + usage = response.json()["usage"]
124 + assert usage["prompt_tokens"] == 10
125 + assert usage["completion_tokens"] == len(synthetic_ids)
126 + assert usage["total_tokens"] == 10 + len(synthetic_ids)
127 +
128 +
129 +@pytest.mark.asyncio
130 +async def test_derender_chat_usage_default(client):
131 + """Omitting prompt_tokens gives usage.prompt_tokens == 0."""
132 + gen_req = await _render_chat(client)
133 + synthetic_ids = gen_req["token_ids"][:3]
134 +
135 + response = await client.post(
136 + "/v1/chat/completions/derender",
137 + json={
138 + "model": MODEL_NAME,
139 + "generate_response": _make_generate_response(synthetic_ids),
140 + },
141 + )
142 + assert response.status_code == 200
143 + usage = response.json()["usage"]
144 + assert usage["prompt_tokens"] == 0
145 +
146 +
147 +@pytest.mark.asyncio
148 +async def test_derender_chat_logprobs(client):
149 + """token_id:N placeholders in content.token are resolved to real strings."""
150 + gen_req = await _render_chat(client)
151 + synthetic_ids = gen_req["token_ids"][:3]
152 + token_id = synthetic_ids[0]
153 +
154 + response = await client.post(
155 + "/v1/chat/completions/derender",
156 + json={
157 + "model": MODEL_NAME,
158 + "generate_response": _make_generate_response(
159 + synthetic_ids,
160 + logprobs=_make_logprobs_with_placeholders(token_id),
161 + ),
162 + },
163 + )
164 + assert response.status_code == 200
165 + data = response.json()
166 + logprobs = data["choices"][0]["logprobs"]
167 + assert logprobs is not None
168 + content = logprobs["content"]
169 + assert content is not None and len(content) == 1
170 + token_str = content[0]["token"]
171 + assert not token_str.startswith("token_id:"), (
172 + f"Placeholder was not resolved: {token_str!r}"
173 + )
174 +
175 +
176 +@pytest.mark.asyncio
177 +async def test_derender_chat_logprobs_bytes(client):
178 + """Resolved logprob entries have bytes populated as list[int]."""
179 + gen_req = await _render_chat(client)
180 + synthetic_ids = gen_req["token_ids"][:3]
181 + token_id = synthetic_ids[0]
182 +
183 + response = await client.post(
184 + "/v1/chat/completions/derender",
185 + json={
186 + "model": MODEL_NAME,
187 + "generate_response": _make_generate_response(
188 + synthetic_ids,
189 + logprobs=_make_logprobs_with_placeholders(token_id),
190 + ),
191 + },
192 + )
193 + assert response.status_code == 200
194 + content = response.json()["choices"][0]["logprobs"]["content"]
195 + bytes_field = content[0]["bytes"]
196 + assert isinstance(bytes_field, list)
197 + assert len(bytes_field) > 0
198 + assert all(isinstance(b, int) for b in bytes_field)
199 +
200 +
201 +@pytest.mark.asyncio
202 +async def test_derender_chat_top_logprobs(client):
203 + """top_logprobs entries also have their placeholders resolved."""
204 + gen_req = await _render_chat(client)
205 + synthetic_ids = gen_req["token_ids"][:3]
206 + token_id = synthetic_ids[0]
207 +
208 + response = await client.post(
209 + "/v1/chat/completions/derender",
210 + json={
211 + "model": MODEL_NAME,
212 + "generate_response": _make_generate_response(
213 + synthetic_ids,
214 + logprobs=_make_logprobs_with_placeholders(token_id),
215 + ),
216 + },
217 + )
218 + assert response.status_code == 200
219 + content = response.json()["choices"][0]["logprobs"]["content"]
220 + top = content[0]["top_logprobs"]
221 + assert len(top) == 1
222 + assert not top[0]["token"].startswith("token_id:"), (
223 + f"top_logprobs placeholder not resolved: {top[0]['token']!r}"
224 + )
225 +
226 +
227 +@pytest.mark.asyncio
228 +async def test_derender_chat_prompt_logprobs_passthrough(client):
229 + """prompt_logprobs on GenerateResponse passes through unchanged."""
230 + gen_req = await _render_chat(client)
231 + synthetic_ids = gen_req["token_ids"][:3]
232 + # prompt_logprobs is a list[dict[int, Logprob] | None]; use None entries.
233 + prompt_logprobs = [None, None]
234 +
235 + response = await client.post(
236 + "/v1/chat/completions/derender",
237 + json={
238 + "model": MODEL_NAME,
239 + "generate_response": _make_generate_response(
240 + synthetic_ids, prompt_logprobs=prompt_logprobs
241 + ),
242 + },
243 + )
244 + assert response.status_code == 200
245 + assert response.json()["prompt_logprobs"] == prompt_logprobs
246 +
247 +
248 +@pytest.mark.asyncio
249 +async def test_derender_chat_kv_transfer_params_passthrough(client):
250 + """kv_transfer_params passes through to the ChatCompletionResponse."""
251 + gen_req = await _render_chat(client)
252 + synthetic_ids = gen_req["token_ids"][:3]
253 + kv = {"key": "value"}
254 +
255 + response = await client.post(
256 + "/v1/chat/completions/derender",
257 + json={
258 + "model": MODEL_NAME,
259 + "generate_response": _make_generate_response(
260 + synthetic_ids, kv_transfer_params=kv
261 + ),
262 + },
263 + )
264 + assert response.status_code == 200
265 + assert response.json()["kv_transfer_params"] == kv
266 +
267 +
268 +@pytest.mark.asyncio
269 +async def test_derender_chat_empty_token_ids(client):
270 + """Empty token_ids list returns 400."""
271 + response = await client.post(
272 + "/v1/chat/completions/derender",
273 + json={
274 + "model": MODEL_NAME,
275 + "generate_response": _make_generate_response([]),
276 + },
277 + )
278 + assert response.status_code == 400
279 +
280 +
281 +@pytest.mark.asyncio
282 +async def test_derender_chat_null_token_ids(client):
283 + """Null token_ids returns 400."""
284 + response = await client.post(
285 + "/v1/chat/completions/derender",
286 + json={
287 + "model": MODEL_NAME,
288 + "generate_response": _make_generate_response(None),
289 + },
290 + )
291 + assert response.status_code == 400
292 +
293 +
294 +@pytest.mark.asyncio
295 +async def test_derender_chat_unknown_model(client):
296 + """Unknown model returns 404."""
297 + gen_req = await _render_chat(client)
298 + synthetic_ids = gen_req["token_ids"][:3]
299 +
300 + response = await client.post(
301 + "/v1/chat/completions/derender",
302 + json={
303 + "model": "does-not-exist",
304 + "generate_response": _make_generate_response(synthetic_ids),
305 + },
306 + )
307 + assert response.status_code == 404
308 +
309 +
310 +# ---------------------------------------------------------------------------
311 +# Completion derender tests
312 +# ---------------------------------------------------------------------------
313 +
314 +
315 +async def _render_completion(client: httpx.AsyncClient, prompt: str) -> dict:
316 + """Render a completion prompt and return the first GenerateRequest dict."""
317 + resp = await client.post(
318 + "/v1/completions/render",
319 + json={"model": MODEL_NAME, "prompt": prompt},
320 + )
321 + assert resp.status_code == 200
322 + data = resp.json()
323 + assert isinstance(data, list) and len(data) >= 1
324 + return data[0]
325 +
326 +
327 +def _make_completion_generate_response(
328 + token_ids: list[int],
329 + request_id: str,
330 + kv_transfer_params: dict | None = None,
331 + logprobs: dict | None = None,
332 +) -> dict:
333 + return {
334 + "request_id": request_id,
335 + "choices": [
336 + {
337 + "index": 0,
338 + "token_ids": token_ids,
339 + "finish_reason": "stop",
340 + "logprobs": logprobs,
341 + }
342 + ],
343 + "prompt_logprobs": None,
344 + "kv_transfer_params": kv_transfer_params,
345 + }
346 +
347 +
348 +@pytest.mark.asyncio
349 +async def test_derender_completion_roundtrip(client):
350 + """Two prompts rendered, two GenerateResponses → two choices with indices 0, 1."""
351 + gr1 = await _render_completion(client, "Hello world")
352 + gr2 = await _render_completion(client, "Goodbye world")
353 +
354 + ids1 = gr1["token_ids"][:4]
355 + ids2 = gr2["token_ids"][:4]
356 +
357 + response = await client.post(
358 + "/v1/completions/derender",
359 + json={
360 + "model": MODEL_NAME,
361 + "generate_responses": [
362 + _make_completion_generate_response(ids1, gr1["request_id"]),
363 + _make_completion_generate_response(ids2, gr2["request_id"]),
364 + ],
365 + },
366 + )
367 + assert response.status_code == 200
368 + data = response.json()
369 + assert data["object"] == "text_completion"
370 + choices = data["choices"]
371 + assert len(choices) == 2
372 + assert choices[0]["index"] == 0
373 + assert choices[1]["index"] == 1
374 + assert choices[0]["text"]
375 + assert choices[1]["text"]
376 +
377 +
378 +@pytest.mark.asyncio
379 +async def test_derender_completion_usage_aggregation(client):
380 + """prompt_tokens=[5, 10] is aggregated correctly into usage."""
381 + gr1 = await _render_completion(client, "Hello")
382 + gr2 = await _render_completion(client, "World")
383 +
384 + ids1 = gr1["token_ids"][:3]
385 + ids2 = gr2["token_ids"][:4]
386 +
387 + response = await client.post(
388 + "/v1/completions/derender",
389 + json={
390 + "model": MODEL_NAME,
391 + "generate_responses": [
392 + _make_completion_generate_response(ids1, gr1["request_id"]),
393 + _make_completion_generate_response(ids2, gr2["request_id"]),
394 + ],
395 + "prompt_tokens": [5, 10],
396 + },
397 + )
398 + assert response.status_code == 200
399 + usage = response.json()["usage"]
400 + assert usage["prompt_tokens"] == 15
401 + assert usage["completion_tokens"] == len(ids1) + len(ids2)
402 + assert usage["total_tokens"] == 15 + len(ids1) + len(ids2)
403 +
404 +
405 +@pytest.mark.asyncio
406 +async def test_derender_completion_prompt_tokens_length_mismatch(client):
407 + """len(prompt_tokens) != len(generate_responses) returns 400."""
408 + gr1 = await _render_completion(client, "Hello")
409 + ids1 = gr1["token_ids"][:3]
410 +
411 + response = await client.post(
412 + "/v1/completions/derender",
413 + json={
414 + "model": MODEL_NAME,
415 + "generate_responses": [
416 + _make_completion_generate_response(ids1, gr1["request_id"]),
417 + ],
418 + "prompt_tokens": [5, 10],
419 + },
420 + )
421 + assert response.status_code == 400
422 +
423 +
424 +@pytest.mark.asyncio
425 +async def test_derender_completion_empty_generate_responses(client):
426 + """Empty generate_responses list returns 400."""
427 + response = await client.post(
428 + "/v1/completions/derender",
429 + json={"model": MODEL_NAME, "generate_responses": []},
430 + )
431 + assert response.status_code == 400
432 +
433 +
434 +@pytest.mark.asyncio
435 +async def test_derender_completion_logprobs(client):
436 + """token_id:N placeholders in logprobs are resolved; CompletionLogProbs
437 + flat-list structure is returned with non-empty tokens and text_offsets."""
438 + gr1 = await _render_completion(client, "Hello world")
439 + ids1 = gr1["token_ids"][:3]
440 + token_id = ids1[0]
441 +
442 + response = await client.post(
443 + "/v1/completions/derender",
444 + json={
445 + "model": MODEL_NAME,
446 + "generate_responses": [
447 + _make_completion_generate_response(
448 + ids1,
449 + gr1["request_id"],
450 + logprobs=_make_logprobs_with_placeholders(token_id),
451 + ),
452 + ],
453 + },
454 + )
455 + assert response.status_code == 200
456 + logprobs = response.json()["choices"][0]["logprobs"]
457 + assert logprobs is not None
458 + tokens = logprobs["tokens"]
459 + assert len(tokens) == 1
460 + assert not tokens[0].startswith("token_id:"), (
461 + f"Placeholder was not resolved: {tokens[0]!r}"
462 + )
463 + assert len(logprobs["token_logprobs"]) == 1
464 + assert isinstance(logprobs["token_logprobs"][0], float)
465 + assert len(logprobs["text_offset"]) == 1
466 + assert logprobs["text_offset"][0] == 0
467 +
468 +
469 +@pytest.mark.asyncio
470 +async def test_derender_completion_kv_transfer_params_passthrough(client):
471 + """kv_transfer_params passes through to CompletionResponse."""
472 + gr1 = await _render_completion(client, "Hello")
473 + ids1 = gr1["token_ids"][:3]
474 + kv = {"node": "abc"}
475 +
476 + response = await client.post(
477 + "/v1/completions/derender",
478 + json={
479 + "model": MODEL_NAME,
480 + "generate_responses": [
481 + _make_completion_generate_response(
482 + ids1, gr1["request_id"], kv_transfer_params=kv
483 + ),
484 + ],
485 + },
486 + )
487 + assert response.status_code == 200
488 + assert response.json()["kv_transfer_params"] == kv
vllm/entrypoints/openai/chat_completion/serving.py
+2 -1
@@ -46,6 +46,7 @@ from vllm.entrypoints.openai.engine.serving import (
46 GenerationError,
47 OpenAIServing,
48 clamp_prompt_logprobs,
49 + format_token_id_placeholder,
50 )
51 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
52 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
@@ -1129,7 +1130,7 @@ class OpenAIServingChat(OpenAIServing):
1130 step_top_logprobs = top_logprobs[i]
1131 if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1132 if should_return_as_token_id:
1132 - token = f"token_id:{token_id}"
1133 + token = format_token_id_placeholder(token_id)
1134 else:
1135 if tokenizer is None:
1136 raise ValueError(
vllm/entrypoints/openai/completion/serving.py
+2 -1
@@ -31,6 +31,7 @@ from vllm.entrypoints.openai.engine.serving import (
31 GenerationError,
32 OpenAIServing,
33 clamp_prompt_logprobs,
34 + format_token_id_placeholder,
35 )
36 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
37 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
@@ -628,7 +629,7 @@ class OpenAIServingCompletion(OpenAIServing):
629 step_top_logprobs = top_logprobs[i]
630 if step_top_logprobs is None:
631 if should_return_as_token_id:
631 - token = f"token_id:{token_id}"
632 + token = format_token_id_placeholder(token_id)
633 else:
634 if tokenizer is None:
635 raise VLLMValidationError(
vllm/entrypoints/openai/engine/serving.py
+33 -1
@@ -452,7 +452,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
452 return_as_token_id: bool = False,
453 ) -> str:
454 if return_as_token_id:
455 - return f"token_id:{token_id}"
455 + return format_token_id_placeholder(token_id)
456
457 if logprob.decoded_token is not None:
458 return logprob.decoded_token
@@ -472,6 +472,38 @@ class OpenAIServing(BeamSearchOnlineMixin):
472 return self.models.is_base_model(model_name)
473
474
475 +def format_token_id_placeholder(token_id: int) -> str:
476 + return f"token_id:{token_id}"
477 +
478 +
479 +def resolve_token_id_placeholder(
480 + token: str, tokenizer: TokenizerLike
481 +) -> tuple[str, list[int] | None]:
482 + """Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
483 +
484 + Returns (token, None) unchanged if token is not a placeholder.
485 + This is the inverse of format_token_id_placeholder / _get_decoded_token
486 + when return_as_token_id=True.
487 + """
488 + suffix = token.removeprefix("token_id:")
489 + if suffix == token:
490 + return token, None
491 + try:
492 + token_id = int(suffix)
493 + except ValueError:
494 + return token, None
495 + token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
496 + if token_repr is None:
497 + logger.warning_once(
498 + "resolve_token_id_placeholder: token_id %d has no vocab entry; "
499 + "substituting empty string",
500 + token_id,
501 + )
502 + return "", None
503 + token_str = tokenizer.convert_tokens_to_string([token_repr])
504 + return token_str, list(token_str.encode("utf-8", errors="replace"))
505 +
506 +
507 def clamp_prompt_logprobs(
508 prompt_logprobs: PromptLogprobs | None,
509 ) -> PromptLogprobs | None:
vllm/entrypoints/serve/disagg/protocol.py
+68 -1
@@ -11,7 +11,11 @@ from pydantic import (
11 )
12
13 from vllm.config import ModelConfig
14 -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
14 +from vllm.entrypoints.openai.chat_completion.protocol import (
15 + ChatCompletionLogProbs,
16 + ChatCompletionRequest,
17 +)
18 +from vllm.entrypoints.openai.completion.protocol import CompletionRequest
19 from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
20 from vllm.logprobs import Logprob
21 from vllm.renderers import TokenizeParams
@@ -209,3 +213,66 @@ class GenerateResponse(BaseModel):
213 default=None,
214 description="KVTransfer parameters used for disaggregated serving.",
215 )
216 +
217 +
218 +####### Derender (postprocessing) #######
219 +
220 +
221 +class DerenderChatRequest(BaseModel):
222 + """Request for the /v1/chat/completions/derender endpoint.
223 +
224 + Wraps a GenerateResponse and caller-supplied metadata needed to produce
225 + a fully-formed ChatCompletionResponse without a GPU.
226 + """
227 +
228 + model: str
229 + generate_response: GenerateResponse
230 + prompt_tokens: int | None = None
231 + """Prompt token count for usage; defaults to 0 if omitted.
232 +
233 + GenerateResponse carries only output tokens; the caller already has
234 + len(GenerateRequest.token_ids) from the render step.
235 + """
236 +
237 + chat_request: ChatCompletionRequest | None = None
238 + """The original (post-adjust_request) ChatCompletionRequest from /render.
239 +
240 + Required by the parsing so that tool/reasoning parsers can receive the full
241 + request context they expect (request.tools, request.tool_choice,
242 + request._grammar_from_tool_parser, etc.).
243 + """
244 +
245 +
246 +class DerenderCompletionRequest(BaseModel):
247 + """Request for the /v1/completions/derender endpoint.
248 +
249 + Parallel to DerenderChatRequest but handles the multi-prompt completions
250 + case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
251 + returned by /v1/completions/render.
252 + """
253 +
254 + model: str
255 + generate_responses: list[GenerateResponse]
256 + prompt_tokens: list[int] | None = None
257 + """One prompt token count per response; each defaults to 0 if omitted.
258 +
259 + If provided, len(prompt_tokens) must equal len(generate_responses).
260 + """
261 +
262 + completion_request: CompletionRequest | None = None
263 + """The original (post-adjust_request) CompletionRequest from /render.
264 +
265 + Mirrors chat_request on DerenderChatRequest. Required by the parsing
266 + so parsers receive the full request context.
267 + """
268 +
269 + @model_validator(mode="after")
270 + def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest":
271 + if self.prompt_tokens is not None and len(self.prompt_tokens) != len(
272 + self.generate_responses
273 + ):
274 + raise ValueError(
275 + f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
276 + f"generate_responses length ({len(self.generate_responses)})"
277 + )
278 + return self
vllm/entrypoints/serve/render/api_router.py
+61 -3
@@ -5,10 +5,20 @@ from http import HTTPStatus
5 from fastapi import APIRouter, Depends, FastAPI, Request
6 from fastapi.responses import JSONResponse
7
8 -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9 -from vllm.entrypoints.openai.completion.protocol import CompletionRequest
8 +from vllm.entrypoints.openai.chat_completion.protocol import (
9 + ChatCompletionRequest,
10 + ChatCompletionResponse,
11 +)
12 +from vllm.entrypoints.openai.completion.protocol import (
13 + CompletionRequest,
14 + CompletionResponse,
15 +)
16 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11 -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
17 +from vllm.entrypoints.serve.disagg.protocol import (
18 + DerenderChatRequest,
19 + DerenderCompletionRequest,
20 + GenerateRequest,
21 +)
22 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
23 from vllm.entrypoints.serve.utils.api_utils import validate_json_request
24 from vllm.logger import init_logger
@@ -71,5 +81,53 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
81 return JSONResponse(content=[item.model_dump() for item in result])
82
83
84 +@router.post(
85 + "/v1/chat/completions/derender",
86 + dependencies=[Depends(validate_json_request)],
87 + response_model=ChatCompletionResponse,
88 + responses={
89 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
90 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
91 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
92 + },
93 +)
94 +async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
95 + handler = render(raw_request)
96 + if handler is None:
97 + raise NotImplementedError(
98 + "The model does not support Chat Completions Derender API"
99 + )
100 +
101 + result = await handler.derender_chat_response(request)
102 +
103 + if isinstance(result, ErrorResponse):
104 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
105 +
106 + return JSONResponse(content=result.model_dump())
107 +
108 +
109 +@router.post(
110 + "/v1/completions/derender",
111 + dependencies=[Depends(validate_json_request)],
112 + response_model=CompletionResponse,
113 + responses={
114 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
115 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
116 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
117 + },
118 +)
119 +async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
120 + handler = render(raw_request)
121 + if handler is None:
122 + raise NotImplementedError("The model does not support Completions Derender API")
123 +
124 + result = await handler.derender_completion_response(request)
125 +
126 + if isinstance(result, ErrorResponse):
127 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
128 +
129 + return JSONResponse(content=result.model_dump())
130 +
131 +
132 def attach_router(app: FastAPI) -> None:
133 app.include_router(router)
vllm/entrypoints/serve/render/serving.py
+244 -2
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import time
4 from collections.abc import Sequence
5 from http import HTTPStatus
6 from typing import Any, cast
@@ -11,11 +12,24 @@ from vllm.entrypoints.chat_utils import (
12 ChatTemplateContentFormatOption,
13 ConversationMessage,
14 )
14 -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
15 -from vllm.entrypoints.openai.completion.protocol import CompletionRequest
15 +from vllm.entrypoints.openai.chat_completion.protocol import (
16 + ChatCompletionLogProbs,
17 + ChatCompletionRequest,
18 + ChatCompletionResponse,
19 + ChatCompletionResponseChoice,
20 + ChatMessage,
21 +)
22 +from vllm.entrypoints.openai.completion.protocol import (
23 + CompletionLogProbs,
24 + CompletionRequest,
25 + CompletionResponse,
26 + CompletionResponseChoice,
27 +)
28 from vllm.entrypoints.openai.engine.protocol import (
29 ErrorResponse,
30 + UsageInfo,
31 )
32 +from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
33 from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
34 from vllm.entrypoints.openai.parser.harmony_utils import (
35 build_harmony_preamble,
@@ -26,7 +40,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
40 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
41 from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
42 from vllm.entrypoints.serve.disagg.protocol import (
43 + DerenderChatRequest,
44 + DerenderCompletionRequest,
45 GenerateRequest,
46 + GenerateResponseChoice,
47 MultiModalFeatures,
48 PlaceholderRangeInfo,
49 )
@@ -51,6 +68,7 @@ from vllm.renderers.inputs.preprocess import (
68 parse_model_prompt,
69 prompt_to_seq,
70 )
71 +from vllm.tokenizers import TokenizerLike
72 from vllm.utils import random_uuid
73 from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
74 from vllm.utils.mistral import mt as _mt
@@ -58,6 +76,90 @@ from vllm.utils.mistral import mt as _mt
76 logger = init_logger(__name__)
77
78
79 +def _resolve_logprobs(
80 + logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
81 +) -> ChatCompletionLogProbs:
82 + """Resolve all token_id:N placeholders in a ChatCompletionLogProbs object."""
83 + if logprobs.content is None:
84 + return logprobs
85 + resolved_content = []
86 + for entry in logprobs.content:
87 + token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
88 + resolved_top = []
89 + for top in entry.top_logprobs:
90 + top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
91 + resolved_top.append(
92 + top.model_copy(update={"token": top_str, "bytes": top_bytes})
93 + )
94 + resolved_content.append(
95 + entry.model_copy(
96 + update={
97 + "token": token_str,
98 + "bytes": token_bytes,
99 + "top_logprobs": resolved_top,
100 + }
101 + )
102 + )
103 + return ChatCompletionLogProbs(content=resolved_content)
104 +
105 +
106 +def _convert_chat_logprobs_to_completion_logprobs(
107 + logprobs: ChatCompletionLogProbs,
108 +) -> CompletionLogProbs:
109 + """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
110 + (parallel flat lists) as required by the /v1/completions response schema."""
111 + if logprobs.content is None:
112 + return CompletionLogProbs()
113 +
114 + tokens: list[str] = []
115 + token_logprobs: list[float | None] = []
116 + top_logprobs_list: list[dict[str, float] | None] = []
117 + text_offset: list[int] = []
118 +
119 + offset = 0
120 + for entry in logprobs.content:
121 + text_offset.append(offset)
122 + tokens.append(entry.token)
123 + token_logprobs.append(entry.logprob)
124 + top_logprobs_list.append(
125 + {t.token: t.logprob for t in entry.top_logprobs}
126 + if entry.top_logprobs
127 + else None
128 + )
129 + offset += len(entry.token)
130 +
131 + return CompletionLogProbs(
132 + text_offset=text_offset,
133 + token_logprobs=token_logprobs,
134 + tokens=tokens,
135 + top_logprobs=top_logprobs_list,
136 + )
137 +
138 +
139 +def _build_chat_choice(
140 + choice: GenerateResponseChoice, tokenizer: TokenizerLike
141 +) -> ChatCompletionResponseChoice:
142 + """Detokenize and resolve logprobs for a single GenerateResponseChoice.
143 +
144 + Raises:
145 + ValueError: if choice.token_ids is empty or None.
146 + """
147 + if not choice.token_ids:
148 + raise ValueError(f"choice {choice.index} has empty or null token_ids")
149 + decoded_text = tokenizer.decode(choice.token_ids, skip_special_tokens=True)
150 + resolved_logprobs = (
151 + _resolve_logprobs(choice.logprobs, tokenizer)
152 + if choice.logprobs is not None
153 + else None
154 + )
155 + return ChatCompletionResponseChoice(
156 + index=choice.index,
157 + message=ChatMessage(role="assistant", content=decoded_text),
158 + logprobs=resolved_logprobs,
159 + finish_reason=choice.finish_reason,
160 + )
161 +
162 +
163 class OpenAIServingRender:
164 def __init__(
165 self,
@@ -427,6 +529,146 @@ class OpenAIServingRender:
529
530 return messages, [engine_input]
531
532 + async def derender_chat_response(
533 + self,
534 + request: DerenderChatRequest,
535 + ) -> ChatCompletionResponse | ErrorResponse:
536 + """Postprocess a GenerateResponse into a ChatCompletionResponse.
537 +
538 + This is the symmetric inverse of render_chat_request: it detokenizes
539 + output token IDs, resolves token_id:N logprob placeholders, and
540 + formats the result as an OpenAI-compatible chat completion response.
541 + """
542 + error_check_ret = await self._check_model(request)
543 + if error_check_ret is not None:
544 + return error_check_ret
545 +
546 + tokenizer = self.renderer.get_tokenizer()
547 + gen = request.generate_response
548 + choices: list[ChatCompletionResponseChoice] = []
549 +
550 + try:
551 + for choice in gen.choices:
552 + choices.append(_build_chat_choice(choice, tokenizer))
553 + except ValueError as exc:
554 + return self.create_error_response(str(exc))
555 +
556 + prompt_tokens = (
557 + request.prompt_tokens if request.prompt_tokens is not None else 0
558 + )
559 + completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
560 + usage = UsageInfo(
561 + prompt_tokens=prompt_tokens,
562 + completion_tokens=completion_tokens,
563 + total_tokens=prompt_tokens + completion_tokens,
564 + )
565 +
566 + logger.debug(
567 + "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
568 + gen.request_id,
569 + request.model,
570 + len(choices),
571 + completion_tokens,
572 + )
573 + return ChatCompletionResponse(
574 + id=gen.request_id,
575 + model=request.model,
576 + created=int(time.time()),
577 + choices=choices,
578 + usage=usage,
579 + prompt_logprobs=gen.prompt_logprobs,
580 + kv_transfer_params=gen.kv_transfer_params,
581 + )
582 +
583 + async def derender_completion_response(
584 + self,
585 + request: DerenderCompletionRequest,
586 + ) -> CompletionResponse | ErrorResponse:
587 + """Postprocess a list of GenerateResponses into a CompletionResponse.
588 +
589 + Mirrors the multi-prompt completions case: one GenerateResponse per
590 + prompt, parallel to the list[GenerateRequest] from /v1/completions/render.
591 + """
592 + error_check_ret = await self._check_model(request)
593 + if error_check_ret is not None:
594 + return error_check_ret
595 +
596 + n = len(request.generate_responses)
597 + prompt_tokens_list: list[int] = (
598 + request.prompt_tokens if request.prompt_tokens is not None else [0] * n
599 + )
600 +
601 + tokenizer = self.renderer.get_tokenizer()
602 + choices: list[CompletionResponseChoice] = []
603 + total_prompt_tokens = 0
604 + total_completion_tokens = 0
605 + index = 0
606 +
607 + for gen, pt in zip(request.generate_responses, prompt_tokens_list):
608 + for choice in gen.choices:
609 + if not choice.token_ids:
610 + return self.create_error_response(
611 + f"choice {choice.index} in response {gen.request_id} "
612 + "has empty or null token_ids"
613 + )
614 + decoded_text = tokenizer.decode(
615 + choice.token_ids, skip_special_tokens=True
616 + )
617 + completion_logprobs = None
618 + if choice.logprobs is not None:
619 + resolved = _resolve_logprobs(choice.logprobs, tokenizer)
620 + completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
621 + resolved
622 + )
623 + choices.append(
624 + CompletionResponseChoice(
625 + index=index,
626 + text=decoded_text,
627 + finish_reason=choice.finish_reason,
628 + logprobs=completion_logprobs,
629 + )
630 + )
631 + total_completion_tokens += len(choice.token_ids)
632 + index += 1
633 + total_prompt_tokens += pt
634 +
635 + if not request.generate_responses:
636 + return self.create_error_response("generate_responses must not be empty")
637 +
638 + first = request.generate_responses[0]
639 + kv_params = first.kv_transfer_params
640 + if any(
641 + r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
642 + ):
643 + logger.warning(
644 + "derender_completion: kv_transfer_params differ across responses; "
645 + "setting to None on the aggregated response"
646 + )
647 + kv_params = None
648 +
649 + usage = UsageInfo(
650 + prompt_tokens=total_prompt_tokens,
651 + completion_tokens=total_completion_tokens,
652 + total_tokens=total_prompt_tokens + total_completion_tokens,
653 + )
654 +
655 + logger.debug(
656 + "derender_completion request_id=%s model=%s choices=%d"
657 + " completion_tokens=%d",
658 + first.request_id,
659 + request.model,
660 + len(choices),
661 + total_completion_tokens,
662 + )
663 + return CompletionResponse(
664 + id=first.request_id,
665 + model=request.model,
666 + created=int(time.time()),
667 + choices=choices,
668 + usage=usage,
669 + kv_transfer_params=kv_params,
670 + )
671 +
672 def create_error_response(
673 self,
674 message: str | Exception,