@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 3483240b7e

[Frontend] Consolidate scale out entrypoints (#44512)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

wang.yuqi committed Jun 29, 2026 at 18:18 UTC 3483240b7ea3d4372b6c79369ea36617f8b1fbb2
45 files changed +422 -289
.buildkite/test-amd.yaml
+4
@@ -845,10 +845,12 @@ steps:
845 source_file_dependencies:
846 - vllm/
847 - tests/entrypoints/serve
848 + - tests/entrypoints/scale_out
849 commands:
850 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
851 - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
852 - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
853 + - pytest -v -s entrypoints/scale_out
854
855 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
856 timeout_in_minutes: 180
@@ -2559,10 +2561,12 @@ steps:
2561 source_file_dependencies:
2562 - vllm/
2563 - tests/entrypoints/serve
2564 + - tests/entrypoints/scale_out
2565 commands:
2566 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2567 - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
2568 - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
2569 + - pytest -v -s entrypoints/scale_out
2570
2571 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
2572 timeout_in_minutes: 180
.buildkite/test_areas/entrypoints.yaml
+2
@@ -40,10 +40,12 @@ steps:
40 source_file_dependencies:
41 - vllm/
42 - tests/entrypoints/serve
43 + - tests/entrypoints/scale_out
44 commands:
45 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
46 - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
47 - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
48 + - pytest -v -s entrypoints/scale_out
49 mirror:
50 amd:
51 device: mi325_1
.buildkite/test_areas/rust_frontend.yaml
+2 -2
@@ -46,7 +46,7 @@ steps:
46 - vllm/v1/engine/
47 - tests/utils.py
48 # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
49 - - tests/entrypoints/serve/disagg/test_serving_tokens.py
49 + - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
50 - tests/entrypoints/serve/instrumentator/test_basic.py
51 - tests/entrypoints/serve/instrumentator/test_metrics.py
52 # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
55 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
56 # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
57 - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
58 - - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
58 + - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
59 - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
60 # - pytest -v -s entrypoints/serve/dev/test_sleep.py
61
docs/examples/README.md
+2 -1
@@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories:
13 - **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
14 - **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
15 - **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
16 -- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
16 +- **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
17 +- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server.
18 - **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).
docs/serving/online_serving/README.md
+2 -2
@@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
119 - `/ping` - SageMaker health check
120 - `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
121
122 -## Disaggregated Everything
122 +## Scale-Out APIs
123
124 -### Tokens IN <> Tokens OUT
124 +### Tokens IN <> Tokens OUT APIs
125
126 - `/inference/v1/generate` - Generate completions
127 - `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
examples/__init__.py renamed
examples/scale_out/__init__.py renamed
examples/scale_out/example_mm_serve.py renamed
examples/scale_out/token_generation_client.py renamed
tests/entrypoints/openai/chat_completion/test_chat_error.py
+2 -10
@@ -17,10 +17,9 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
17 from vllm.entrypoints.openai.engine.protocol import GenerationError
18 from vllm.entrypoints.openai.models.protocol import BaseModelPath
19 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
20 -from vllm.entrypoints.serve.render.serving import ServingRender
20 +from vllm.entrypoints.scale_out.render.serving import ServingRender
21 from vllm.outputs import CompletionOutput, RequestOutput
22 from vllm.renderers.hf import HfRenderer
23 -from vllm.renderers.online_derenderer import OnlineDerenderer
23 from vllm.renderers.online_renderer import OnlineRenderer
24 from vllm.tokenizers.registry import cached_tokenizer_from_config
25 from vllm.v1.engine.async_llm import AsyncLLM
@@ -206,15 +205,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
205 chat_template=None,
206 chat_template_content_format="auto",
207 )
209 - online_derenderer = OnlineDerenderer(
210 - model_config=engine.model_config,
211 - renderer=engine.renderer,
212 - request_logger=None,
213 - chat_template=None,
214 - chat_template_content_format="auto",
215 - )
208
217 - serving_render = ServingRender(models, online_renderer, online_derenderer)
209 + serving_render = ServingRender(models, online_renderer)
210
211 async def _fake_preprocess_chat(*args, **kwargs):
212 # return conversation, engine_inputs
tests/entrypoints/openai/completion/test_completion_error.py
+2 -10
@@ -14,10 +14,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14 from vllm.entrypoints.openai.engine.protocol import GenerationError
15 from vllm.entrypoints.openai.models.protocol import BaseModelPath
16 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17 -from vllm.entrypoints.serve.render.serving import ServingRender
17 +from vllm.entrypoints.scale_out.render.serving import ServingRender
18 from vllm.outputs import CompletionOutput, RequestOutput
19 from vllm.renderers.hf import HfRenderer
20 -from vllm.renderers.online_derenderer import OnlineDerenderer
20 from vllm.renderers.online_renderer import OnlineRenderer
21 from vllm.tokenizers.registry import cached_tokenizer_from_config
22 from vllm.v1.engine.async_llm import AsyncLLM
@@ -191,15 +190,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
190 chat_template=None,
191 chat_template_content_format="auto",
192 )
194 - online_derenderer = OnlineDerenderer(
195 - model_config=engine.model_config,
196 - renderer=engine.renderer,
197 - request_logger=None,
198 - chat_template=None,
199 - chat_template_content_format="auto",
200 - )
193
202 - serving_render = ServingRender(models, online_renderer, online_derenderer)
194 + serving_render = ServingRender(models, online_renderer)
195
196 async def _fake_preprocess_chat(*args, **kwargs):
197 # return conversation, engine_inputs
tests/entrypoints/openai/test_render_token_offsets.py
+2 -2
@@ -3,7 +3,7 @@
3 """Unit tests for the token-offsets request/response protocol wiring:
4 the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
5 serialization boundary. End-to-end behavior is covered by
6 -``tests/entrypoints/serve/render/test_render.py``; plain Pydantic field
6 +``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
7 storage is not retested here.
8 """
9
@@ -12,7 +12,7 @@ from unittest.mock import Mock
12 from vllm.config import ModelConfig
13 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
14 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
15 -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
15 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
16 from vllm.sampling_params import SamplingParams
17
18
tests/entrypoints/scale_out/__init__.py renamed
tests/entrypoints/scale_out/derender/__init__.py
tests/entrypoints/scale_out/derender/test_derender.py renamed
tests/entrypoints/scale_out/render/__init__.py
tests/entrypoints/scale_out/render/test_launch_render.py renamed
tests/entrypoints/scale_out/render/test_render.py renamed
tests/entrypoints/scale_out/render/test_render_multimodal.py renamed
tests/entrypoints/scale_out/token_in_token_out/__init__.py
tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py renamed
+2 -2
@@ -12,11 +12,11 @@ from vllm.config.multimodal import MultiModalConfig
12 from vllm.entrypoints.openai.engine.protocol import StreamOptions
13 from vllm.entrypoints.openai.models.protocol import BaseModelPath
14 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15 -from vllm.entrypoints.serve.disagg.protocol import (
15 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
16 GenerateRequest,
17 GenerateResponse,
18 )
19 -from vllm.entrypoints.serve.disagg.serving import ServingTokens
19 +from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
20 from vllm.logprobs import Logprob
21 from vllm.outputs import CompletionOutput, RequestOutput
22 from vllm.renderers import renderer_from_config
tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py renamed
+6 -3
@@ -1,14 +1,17 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Roundtrip tests for multimodal serde used by the disagg generate endpoint."""
3 +"""
4 +Roundtrip tests for multimodal serde used by the
5 +token_in_token_out generate endpoint.
6 +"""
7
8 import torch
9
7 -from vllm.entrypoints.serve.disagg.mm_serde import (
10 +from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
11 decode_mm_kwargs_item,
12 encode_mm_kwargs_item,
13 )
11 -from vllm.entrypoints.serve.disagg.protocol import (
14 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
15 MultiModalFeatures,
16 PlaceholderRangeInfo,
17 )
tests/entrypoints/scale_out/token_in_token_out/test_protocol.py renamed
+2 -2
@@ -1,6 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Unit tests for the disagg request/response protocol.
3 +"""Unit tests for the token_in_token_out request/response protocol.
4
5 These tests intentionally avoid spinning up a server — they exercise the
6 pydantic validators on ``GenerateRequest`` directly so they run fast and
@@ -9,7 +9,7 @@ fail loudly if the validator semantics ever drift.
9
10 import json
11
12 -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
12 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
13 from vllm.sampling_params import SamplingParams
14
15
tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py renamed
tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py renamed
tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py renamed
tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py renamed
+1 -1
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -from vllm.entrypoints.serve.disagg.serving import ServingTokens
4 +from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
5 from vllm.logprobs import Logprob
6
7
tests/renderers/test_token_offsets.py
+1 -1
@@ -5,7 +5,7 @@
5 These exercise ``_tokenize_prompt`` (offset extraction + capability/MM
6 gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput``
7 forwarding chain. Endpoint-level coverage lives in
8 -``tests/entrypoints/serve/render/test_render.py``.
8 +``tests/entrypoints/scale_out/render/test_render.py``.
9 """
10
11 import pytest
vllm/entrypoints/generate/api_router.py
-15
@@ -66,7 +66,6 @@ async def init_generate_state(
66 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
67 from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
68 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
69 - from vllm.entrypoints.serve.disagg.serving import ServingTokens
69 from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
70
71 # Applied before any serving class is constructed so that each one picks
@@ -175,20 +174,6 @@ async def init_generate_state(
174 if "generate" in supported_tasks
175 else None
176 )
178 - state.serving_tokens = (
179 - ServingTokens(
180 - engine_client,
181 - state.openai_serving_models,
182 - state.online_renderer,
183 - request_logger=request_logger,
184 - return_tokens_as_token_ids=args.return_tokens_as_token_ids,
185 - enable_prompt_tokens_details=args.enable_prompt_tokens_details,
186 - enable_log_outputs=args.enable_log_outputs,
187 - force_no_detokenize=args.tokens_only,
188 - )
189 - if "generate" in supported_tasks
190 - else None
191 - )
177
178 from .generative_scoring.serving import ServingGenerativeScoring
179
vllm/entrypoints/openai/api_server.py
+10 -23
@@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
32 from vllm.entrypoints.openai.models.protocol import BaseModelPath
33 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
34 from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
35 -from vllm.entrypoints.serve.render.serving import ServingRender
35 from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
36 from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
37 from vllm.entrypoints.serve.utils.api_utils import (
@@ -208,12 +207,6 @@ def build_app(
207
208 register_generate_api_routers(app)
209
211 - from vllm.entrypoints.serve.disagg.api_router import (
212 - attach_router as attach_disagg_router,
213 - )
214 -
215 - attach_disagg_router(app)
216 -
210 from vllm.entrypoints.serve.elastic_ep.api_router import (
211 attach_router as elastic_ep_attach_router,
212 )
@@ -221,11 +214,9 @@ def build_app(
214 elastic_ep_attach_router(app)
215
216 if "generate" in supported_tasks or "render" in supported_tasks:
224 - from vllm.entrypoints.serve.render.api_router import (
225 - attach_router as attach_render_router,
226 - )
217 + from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
218
228 - attach_render_router(app)
219 + register_scale_out_api_routers(app, supported_tasks)
220
221 if "transcription" in supported_tasks or "realtime" in supported_tasks:
222 from vllm.entrypoints.speech_to_text.factories import (
@@ -401,12 +392,6 @@ async def init_app_state(
392 default_chat_template_kwargs=args.default_chat_template_kwargs,
393 trust_request_chat_template=args.trust_request_chat_template,
394 )
404 - state.serving_render = ServingRender(
405 - state.openai_serving_models,
406 - state.online_renderer,
407 - state.online_derenderer,
408 - request_logger=request_logger,
409 - )
395
396 if "generate" in supported_tasks:
397 from vllm.entrypoints.generate.api_router import init_generate_state
@@ -415,6 +400,10 @@ async def init_app_state(
400 engine_client, state, args, request_logger, supported_tasks
401 )
402
403 + from vllm.entrypoints.scale_out.factories import init_scale_out_state
404 +
405 + init_scale_out_state(state, args, engine_client, request_logger)
406 +
407 if "transcription" in supported_tasks or "realtime" in supported_tasks:
408 from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
409
@@ -505,12 +494,10 @@ async def init_render_app_state(
494 default_chat_template_kwargs=args.default_chat_template_kwargs,
495 trust_request_chat_template=args.trust_request_chat_template,
496 )
508 - state.serving_render = ServingRender(
509 - model_registry,
510 - state.online_renderer,
511 - state.online_derenderer,
512 - request_logger=request_logger,
513 - )
497 +
498 + from vllm.entrypoints.scale_out.factories import init_render_state
499 +
500 + init_render_state(state, request_logger)
501
502 state.vllm_config = vllm_config
503 # Disable stats logging — there is no engine to poll.
vllm/entrypoints/scale_out/__init__.py
vllm/entrypoints/scale_out/derender/__init__.py
vllm/entrypoints/scale_out/derender/api_router.py new
+74
@@ -0,0 +1,74 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from http import HTTPStatus
4 +
5 +from fastapi import APIRouter, Depends, Request
6 +from fastapi.responses import JSONResponse
7 +
8 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
9 +from vllm.entrypoints.openai.completion.protocol import CompletionResponse
10 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11 +from vllm.entrypoints.serve.utils.api_utils import validate_json_request
12 +from vllm.logger import init_logger
13 +
14 +from ..token_in_token_out.protocol import (
15 + DerenderChatRequest,
16 + DerenderCompletionRequest,
17 +)
18 +from .serving import ServingDerender
19 +
20 +logger = init_logger(__name__)
21 +
22 +router = APIRouter()
23 +
24 +
25 +def derender(request: Request) -> ServingDerender | None:
26 + return getattr(request.app.state, "serving_derender", None)
27 +
28 +
29 +@router.post(
30 + "/v1/chat/completions/derender",
31 + dependencies=[Depends(validate_json_request)],
32 + response_model=ChatCompletionResponse,
33 + responses={
34 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
35 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
36 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
37 + },
38 +)
39 +async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
40 + handler = derender(raw_request)
41 + if handler is None:
42 + raise NotImplementedError(
43 + "The model does not support Chat Completions Derender API"
44 + )
45 +
46 + result = await handler.derender_chat_response(request)
47 +
48 + if isinstance(result, ErrorResponse):
49 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
50 +
51 + return JSONResponse(content=result.model_dump())
52 +
53 +
54 +@router.post(
55 + "/v1/completions/derender",
56 + dependencies=[Depends(validate_json_request)],
57 + response_model=CompletionResponse,
58 + responses={
59 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
60 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
61 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
62 + },
63 +)
64 +async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
65 + handler = derender(raw_request)
66 + if handler is None:
67 + raise NotImplementedError("The model does not support Completions Derender API")
68 +
69 + result = await handler.derender_completion_response(request)
70 +
71 + if isinstance(result, ErrorResponse):
72 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
73 +
74 + return JSONResponse(content=result.model_dump())
vllm/entrypoints/scale_out/derender/serving.py new
+202
@@ -0,0 +1,202 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import time
4 +from typing import cast
5 +
6 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
7 +from vllm.entrypoints.openai.completion.protocol import CompletionResponse
8 +from vllm.entrypoints.openai.engine.protocol import (
9 + ErrorResponse,
10 + UsageInfo,
11 +)
12 +from vllm.entrypoints.openai.models.serving import (
13 + OpenAIModelRegistry,
14 + OpenAIServingModels,
15 +)
16 +from vllm.entrypoints.serve.engine.serving import BaseServing
17 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger
18 +from vllm.inputs import (
19 + EngineInput,
20 + MultiModalHashes,
21 + MultiModalInput,
22 + MultiModalPlaceholders,
23 +)
24 +from vllm.logger import init_logger
25 +from vllm.renderers.online_derenderer import OnlineDerenderer
26 +
27 +from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
28 +from ..token_in_token_out.protocol import (
29 + DerenderChatRequest,
30 + DerenderCompletionRequest,
31 + MultiModalFeatures,
32 + PlaceholderRangeInfo,
33 +)
34 +
35 +logger = init_logger(__name__)
36 +
37 +
38 +class ServingDerender(BaseServing):
39 + def __init__(
40 + self,
41 + models: OpenAIServingModels | OpenAIModelRegistry,
42 + online_derenderer: "OnlineDerenderer",
43 + *,
44 + request_logger: RequestLogger | None = None,
45 + ) -> None:
46 + super().__init__(
47 + models=models,
48 + model_config=models.model_config,
49 + request_logger=request_logger,
50 + )
51 +
52 + self.online_derenderer = online_derenderer
53 +
54 + async def derender_chat_response(
55 + self,
56 + request: DerenderChatRequest,
57 + ) -> ChatCompletionResponse | ErrorResponse:
58 + """Postprocess a GenerateResponse into a ChatCompletionResponse.
59 +
60 + Non-streaming only: expects the complete GenerateResponse with all
61 + token IDs present. Uses ``parser.parse()`` for one-shot extraction.
62 +
63 + When ``request.chat_request`` is provided, the parser splits the
64 + output into (reasoning, content, tool_calls). Otherwise falls
65 + back to plain detokenization.
66 + """
67 + error_check_ret = await self._check_model(request)
68 + if error_check_ret is not None:
69 + return error_check_ret
70 +
71 + try:
72 + choices = await self.online_derenderer.derender_chat(
73 + request.generate_response, request.chat_request
74 + )
75 + except ValueError as exc:
76 + return self.create_error_response(str(exc))
77 +
78 + prompt_tokens = (
79 + request.prompt_tokens if request.prompt_tokens is not None else 0
80 + )
81 + gen = request.generate_response
82 + completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
83 + usage = UsageInfo(
84 + prompt_tokens=prompt_tokens,
85 + completion_tokens=completion_tokens,
86 + total_tokens=prompt_tokens + completion_tokens,
87 + )
88 +
89 + logger.debug(
90 + "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
91 + gen.request_id,
92 + request.model,
93 + len(choices),
94 + completion_tokens,
95 + )
96 + return ChatCompletionResponse(
97 + id=gen.request_id,
98 + model=request.model,
99 + created=int(time.time()),
100 + choices=choices,
101 + usage=usage,
102 + prompt_logprobs=gen.prompt_logprobs,
103 + kv_transfer_params=gen.kv_transfer_params,
104 + )
105 +
106 + async def derender_completion_response(
107 + self,
108 + request: DerenderCompletionRequest,
109 + ) -> CompletionResponse | ErrorResponse:
110 + """Postprocess a list of GenerateResponses into a CompletionResponse.
111 +
112 + Non-streaming only. Mirrors the multi-prompt completions case: one
113 + GenerateResponse per prompt, parallel to the list[GenerateRequest]
114 + from /v1/completions/render.
115 + """
116 + error_check_ret = await self._check_model(request)
117 + if error_check_ret is not None:
118 + return error_check_ret
119 +
120 + (
121 + choices,
122 + total_prompt_tokens,
123 + total_completion_tokens,
124 + ) = await self.online_derenderer.derender_completion(
125 + request.generate_responses, request.prompt_tokens
126 + )
127 +
128 + if not request.generate_responses:
129 + return self.create_error_response("generate_responses must not be empty")
130 +
131 + first = request.generate_responses[0]
132 + kv_params = first.kv_transfer_params
133 + if any(
134 + r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
135 + ):
136 + logger.warning(
137 + "derender_completion: kv_transfer_params differ across responses; "
138 + "setting to None on the aggregated response"
139 + )
140 + kv_params = None
141 +
142 + usage = UsageInfo(
143 + prompt_tokens=total_prompt_tokens,
144 + completion_tokens=total_completion_tokens,
145 + total_tokens=total_prompt_tokens + total_completion_tokens,
146 + )
147 +
148 + logger.debug(
149 + "derender_completion request_id=%s model=%s choices=%d"
150 + " completion_tokens=%d",
151 + first.request_id,
152 + request.model,
153 + len(choices),
154 + total_completion_tokens,
155 + )
156 + return CompletionResponse(
157 + id=first.request_id,
158 + model=request.model,
159 + created=int(time.time()),
160 + choices=choices,
161 + usage=usage,
162 + kv_transfer_params=kv_params,
163 + )
164 +
165 + @staticmethod
166 + def _extract_mm_features(
167 + engine_input: EngineInput,
168 + ) -> MultiModalFeatures | None:
169 + """Extract multimodal metadata from a rendered engine prompt.
170 +
171 + Returns ``None`` for text-only prompts.
172 + """
173 + if engine_input.get("type") != "multimodal":
174 + return None
175 +
176 + # At this point engine_input is a MultiModalInput TypedDict.
177 + mm_engine_input = cast(MultiModalInput, engine_input)
178 + mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
179 + raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
180 +
181 + mm_placeholders = {
182 + modality: [
183 + PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
184 + ]
185 + for modality, ranges in raw_placeholders.items()
186 + }
187 +
188 + # Serialize tensor data per modality.
189 + kwargs_data: dict[str, list[str | None]] | None = None
190 + if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
191 + kwargs_data = {}
192 + for modality, items in raw_mm_kwargs.items():
193 + kwargs_data[modality] = [
194 + encode_mm_kwargs_item(item) if item is not None else None
195 + for item in items
196 + ]
197 +
198 + return MultiModalFeatures(
199 + mm_hashes=mm_hashes,
200 + mm_placeholders=mm_placeholders,
201 + kwargs_data=kwargs_data,
202 + )
vllm/entrypoints/scale_out/factories.py new
+78
@@ -0,0 +1,78 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from argparse import Namespace
4 +from typing import TYPE_CHECKING
5 +
6 +from fastapi import FastAPI
7 +
8 +from vllm.engine.protocol import EngineClient
9 +from vllm.tasks import SupportedTask
10 +
11 +if TYPE_CHECKING:
12 + from starlette.datastructures import State
13 +
14 + from vllm.entrypoints.serve.utils.request_logger import RequestLogger
15 +else:
16 + RequestLogger = object
17 +
18 +
19 +def init_render_state(
20 + state: "State",
21 + request_logger: RequestLogger | None,
22 +):
23 + from .derender.serving import ServingDerender
24 + from .render.serving import ServingRender
25 +
26 + state.serving_render = ServingRender(
27 + state.openai_serving_models,
28 + state.online_renderer,
29 + request_logger=request_logger,
30 + )
31 +
32 + state.serving_derender = ServingDerender(
33 + state.openai_serving_models,
34 + state.online_derenderer,
35 + request_logger=request_logger,
36 + )
37 +
38 +
39 +def init_scale_out_state(
40 + state: "State",
41 + args: "Namespace",
42 + engine_client: "EngineClient",
43 + request_logger: RequestLogger | None,
44 +):
45 + init_render_state(state, request_logger)
46 +
47 + from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
48 +
49 + state.serving_tokens = ServingTokens(
50 + engine_client,
51 + state.openai_serving_models,
52 + state.online_renderer,
53 + request_logger=request_logger,
54 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
55 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
56 + enable_log_outputs=args.enable_log_outputs,
57 + force_no_detokenize=args.tokens_only,
58 + )
59 +
60 +
61 +def register_scale_out_api_routers(
62 + app: FastAPI,
63 + supported_tasks: tuple["SupportedTask", ...],
64 +):
65 + from .render.api_router import router as render_render
66 +
67 + app.include_router(render_render)
68 +
69 + from .derender.api_router import router as derender_render
70 +
71 + app.include_router(derender_render)
72 +
73 + if "generate" in supported_tasks:
74 + from .token_in_token_out.api_router import (
75 + attach_router as attach_disagg_router,
76 + )
77 +
78 + attach_disagg_router(app)
vllm/entrypoints/scale_out/render/__init__.py renamed
vllm/entrypoints/scale_out/render/api_router.py renamed
+6 -67
@@ -2,27 +2,18 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from http import HTTPStatus
4
5 -from fastapi import APIRouter, Depends, FastAPI, Request
5 +from fastapi import APIRouter, Depends, Request
6 from fastapi.responses import JSONResponse
7
8 -from vllm.entrypoints.openai.chat_completion.protocol import (
9 - ChatCompletionRequest,
10 - ChatCompletionResponse,
11 -)
12 -from vllm.entrypoints.openai.completion.protocol import (
13 - CompletionRequest,
14 - CompletionResponse,
15 -)
8 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9 +from vllm.entrypoints.openai.completion.protocol import CompletionRequest
10 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
17 -from vllm.entrypoints.serve.disagg.protocol import (
18 - DerenderChatRequest,
19 - DerenderCompletionRequest,
20 - GenerateRequest,
21 -)
22 -from vllm.entrypoints.serve.render.serving import ServingRender
11 from vllm.entrypoints.serve.utils.api_utils import validate_json_request
12 from vllm.logger import init_logger
13
14 +from ..token_in_token_out.protocol import GenerateRequest
15 +from .serving import ServingRender
16 +
17 logger = init_logger(__name__)
18
19 router = APIRouter()
@@ -79,55 +70,3 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
70 return JSONResponse(content=result.model_dump(), status_code=result.error.code)
71
72 return JSONResponse(content=[item.model_dump() for item in result])
82 -
83 -
84 -@router.post(
85 - "/v1/chat/completions/derender",
86 - dependencies=[Depends(validate_json_request)],
87 - response_model=ChatCompletionResponse,
88 - responses={
89 - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
90 - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
91 - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
92 - },
93 -)
94 -async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
95 - handler = render(raw_request)
96 - if handler is None:
97 - raise NotImplementedError(
98 - "The model does not support Chat Completions Derender API"
99 - )
100 -
101 - result = await handler.derender_chat_response(request)
102 -
103 - if isinstance(result, ErrorResponse):
104 - return JSONResponse(content=result.model_dump(), status_code=result.error.code)
105 -
106 - return JSONResponse(content=result.model_dump())
107 -
108 -
109 -@router.post(
110 - "/v1/completions/derender",
111 - dependencies=[Depends(validate_json_request)],
112 - response_model=CompletionResponse,
113 - responses={
114 - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
115 - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
116 - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
117 - },
118 -)
119 -async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
120 - handler = render(raw_request)
121 - if handler is None:
122 - raise NotImplementedError("The model does not support Completions Derender API")
123 -
124 - result = await handler.derender_completion_response(request)
125 -
126 - if isinstance(result, ErrorResponse):
127 - return JSONResponse(content=result.model_dump(), status_code=result.error.code)
128 -
129 - return JSONResponse(content=result.model_dump())
130 -
131 -
132 -def attach_router(app: FastAPI) -> None:
133 - app.include_router(router)
vllm/entrypoints/scale_out/render/serving.py renamed
+5 -131
@@ -1,28 +1,16 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import time
3 from typing import cast
4
6 -from vllm.entrypoints.openai.chat_completion.protocol import (
7 - ChatCompletionRequest,
8 - ChatCompletionResponse,
9 -)
10 -from vllm.entrypoints.openai.completion.protocol import (
11 - CompletionRequest,
12 - CompletionResponse,
13 -)
14 -from vllm.entrypoints.openai.engine.protocol import (
15 - ErrorResponse,
16 - UsageInfo,
17 -)
5 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
6 +from vllm.entrypoints.openai.completion.protocol import CompletionRequest
7 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse
8 from vllm.entrypoints.openai.models.serving import (
9 OpenAIModelRegistry,
10 OpenAIServingModels,
11 )
22 -from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
23 -from vllm.entrypoints.serve.disagg.protocol import (
24 - DerenderChatRequest,
25 - DerenderCompletionRequest,
12 +from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item
13 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
14 GenerateRequest,
15 MultiModalFeatures,
16 PlaceholderRangeInfo,
@@ -41,7 +29,6 @@ from vllm.renderers.inputs.preprocess import (
29 extract_prompt_components,
30 extract_prompt_len,
31 )
44 -from vllm.renderers.online_derenderer import OnlineDerenderer
32 from vllm.renderers.online_renderer import OnlineRenderer
33 from vllm.utils import random_uuid
34
@@ -53,7 +40,6 @@ class ServingRender(BaseServing):
40 self,
41 models: OpenAIServingModels | OpenAIModelRegistry,
42 online_renderer: "OnlineRenderer",
56 - online_derenderer: "OnlineDerenderer",
43 *,
44 request_logger: RequestLogger | None = None,
45 ) -> None:
@@ -64,7 +50,6 @@ class ServingRender(BaseServing):
50 )
51
52 self.online_renderer = online_renderer
67 - self.online_derenderer = online_derenderer
53
54 self.default_sampling_params = (
55 online_renderer.model_config.get_diff_sampling_param()
@@ -223,117 +208,6 @@ class ServingRender(BaseServing):
208
209 return generate_requests
210
226 - async def derender_chat_response(
227 - self,
228 - request: DerenderChatRequest,
229 - ) -> ChatCompletionResponse | ErrorResponse:
230 - """Postprocess a GenerateResponse into a ChatCompletionResponse.
231 -
232 - Non-streaming only: expects the complete GenerateResponse with all
233 - token IDs present. Uses ``parser.parse()`` for one-shot extraction.
234 -
235 - When ``request.chat_request`` is provided, the parser splits the
236 - output into (reasoning, content, tool_calls). Otherwise falls
237 - back to plain detokenization.
238 - """
239 - error_check_ret = await self._check_model(request)
240 - if error_check_ret is not None:
241 - return error_check_ret
242 -
243 - try:
244 - choices = await self.online_derenderer.derender_chat(
245 - request.generate_response, request.chat_request
246 - )
247 - except ValueError as exc:
248 - return self.create_error_response(str(exc))
249 -
250 - prompt_tokens = (
251 - request.prompt_tokens if request.prompt_tokens is not None else 0
252 - )
253 - gen = request.generate_response
254 - completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
255 - usage = UsageInfo(
256 - prompt_tokens=prompt_tokens,
257 - completion_tokens=completion_tokens,
258 - total_tokens=prompt_tokens + completion_tokens,
259 - )
260 -
261 - logger.debug(
262 - "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
263 - gen.request_id,
264 - request.model,
265 - len(choices),
266 - completion_tokens,
267 - )
268 - return ChatCompletionResponse(
269 - id=gen.request_id,
270 - model=request.model,
271 - created=int(time.time()),
272 - choices=choices,
273 - usage=usage,
274 - prompt_logprobs=gen.prompt_logprobs,
275 - kv_transfer_params=gen.kv_transfer_params,
276 - )
277 -
278 - async def derender_completion_response(
279 - self,
280 - request: DerenderCompletionRequest,
281 - ) -> CompletionResponse | ErrorResponse:
282 - """Postprocess a list of GenerateResponses into a CompletionResponse.
283 -
284 - Non-streaming only. Mirrors the multi-prompt completions case: one
285 - GenerateResponse per prompt, parallel to the list[GenerateRequest]
286 - from /v1/completions/render.
287 - """
288 - error_check_ret = await self._check_model(request)
289 - if error_check_ret is not None:
290 - return error_check_ret
291 -
292 - (
293 - choices,
294 - total_prompt_tokens,
295 - total_completion_tokens,
296 - ) = await self.online_derenderer.derender_completion(
297 - request.generate_responses, request.prompt_tokens
298 - )
299 -
300 - if not request.generate_responses:
301 - return self.create_error_response("generate_responses must not be empty")
302 -
303 - first = request.generate_responses[0]
304 - kv_params = first.kv_transfer_params
305 - if any(
306 - r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
307 - ):
308 - logger.warning(
309 - "derender_completion: kv_transfer_params differ across responses; "
310 - "setting to None on the aggregated response"
311 - )
312 - kv_params = None
313 -
314 - usage = UsageInfo(
315 - prompt_tokens=total_prompt_tokens,
316 - completion_tokens=total_completion_tokens,
317 - total_tokens=total_prompt_tokens + total_completion_tokens,
318 - )
319 -
320 - logger.debug(
321 - "derender_completion request_id=%s model=%s choices=%d"
322 - " completion_tokens=%d",
323 - first.request_id,
324 - request.model,
325 - len(choices),
326 - total_completion_tokens,
327 - )
328 - return CompletionResponse(
329 - id=first.request_id,
330 - model=request.model,
331 - created=int(time.time()),
332 - choices=choices,
333 - usage=usage,
334 - kv_transfer_params=kv_params,
335 - )
336 -
211 @staticmethod
212 def _extract_mm_features(
213 engine_input: EngineInput,
vllm/entrypoints/scale_out/token_in_token_out/__init__.py
vllm/entrypoints/scale_out/token_in_token_out/api_router.py renamed
+6 -7
@@ -11,13 +11,6 @@ from fastapi.responses import JSONResponse, StreamingResponse
11
12 from vllm.engine.protocol import EngineClient
13 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
14 -from vllm.entrypoints.serve.disagg.protocol import (
15 - GenerateRequest,
16 - GenerateResponse,
17 -)
18 -from vllm.entrypoints.serve.disagg.serving import (
19 - ServingTokens,
20 -)
14 from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
15 from vllm.entrypoints.serve.utils.api_utils import (
16 load_aware_call,
@@ -26,6 +19,12 @@ from vllm.entrypoints.serve.utils.api_utils import (
19 )
20 from vllm.logger import init_logger
21
22 +from .protocol import (
23 + GenerateRequest,
24 + GenerateResponse,
25 +)
26 +from .serving import ServingTokens
27 +
28 logger = init_logger(__name__)
29
30
vllm/entrypoints/scale_out/token_in_token_out/mm_serde.py renamed
vllm/entrypoints/scale_out/token_in_token_out/protocol.py renamed
vllm/entrypoints/scale_out/token_in_token_out/serving.py renamed
+9 -8
@@ -28,14 +28,6 @@ from vllm.entrypoints.openai.engine.protocol import (
28 )
29 from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
30 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
31 -from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item
32 -from vllm.entrypoints.serve.disagg.protocol import (
33 - GenerateRequest,
34 - GenerateResponse,
35 - GenerateResponseChoice,
36 - GenerateResponseStreamChoice,
37 - GenerateStreamResponse,
38 -)
31 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
32 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
33 from vllm.inputs import EngineInput, mm_input
@@ -51,6 +43,15 @@ from vllm.renderers.online_renderer import OnlineRenderer
43 from vllm.sampling_params import RequestOutputKind, SamplingParams
44 from vllm.utils.collection_utils import as_list
45
46 +from .mm_serde import decode_mm_kwargs_item
47 +from .protocol import (
48 + GenerateRequest,
49 + GenerateResponse,
50 + GenerateResponseChoice,
51 + GenerateResponseStreamChoice,
52 + GenerateStreamResponse,
53 +)
54 +
55 logger = init_logger(__name__)
56
57
vllm/entrypoints/serve/engine/typing.py
+1 -1
@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.completion.protocol import (
15 CompletionResponse,
16 )
17 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
18 -from vllm.entrypoints.serve.disagg.protocol import (
18 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
19 DerenderChatRequest,
20 DerenderCompletionRequest,
21 GenerateRequest,
vllm/renderers/online_derenderer.py
+1 -1
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.completion.protocol import (
16 )
17 from vllm.entrypoints.openai.engine.protocol import ToolCall
18 from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
19 -from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
19 +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
20 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
21 from vllm.logger import init_logger
22 from vllm.parser import Parser, ParserManager