@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4497431df6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 01:30 UTC

[Frontend] Add GPU-less render serving path (`vllm launch render`) (#36166)

Sage committed Mar 8, 2026 at 17:35 UTC 4497431df654e46fb1fb5e64bf8611e762ae5d87
10 files changed +712 -273
vllm/entrypoints/cli/launch.py
+5 -7
@@ -8,7 +8,7 @@ import uvloop
8 from vllm.engine.arg_utils import AsyncEngineArgs
9 from vllm.entrypoints.cli.types import CLISubcommand
10 from vllm.entrypoints.openai.api_server import (
11 - build_and_serve,
11 + build_and_serve_renderer,
12 setup_server,
13 )
14 from vllm.entrypoints.openai.cli_args import (
@@ -109,19 +109,17 @@ def cmd_init() -> list[CLISubcommand]:
109 async def run_launch_fastapi(args: argparse.Namespace) -> None:
110 """Run the online serving layer with FastAPI (no GPU inference)."""
111 from vllm.config import VllmConfig
112 - from vllm.v1.engine.launch import LaunchEngineClient
112
113 # 1. Socket binding
114 listen_address, sock = setup_server(args)
115
117 - # 2. Create LaunchEngineClient (no GPU)
116 + # 2. Build and serve the API server
117 engine_args = AsyncEngineArgs.from_cli_args(args)
118 model_config = engine_args.create_model_config()
119 vllm_config = VllmConfig(model_config=model_config)
121 - engine_client = LaunchEngineClient.from_vllm_config(vllm_config)
122 -
123 - # 3. Build app, initialize state, and start serving
124 - shutdown_task = await build_and_serve(engine_client, listen_address, sock, args)
120 + shutdown_task = await build_and_serve_renderer(
121 + vllm_config, listen_address, sock, args
122 + )
123 try:
124 await shutdown_task
125 finally:
vllm/entrypoints/openai/api_server.py
+113 -3
@@ -22,6 +22,7 @@ from fastapi.middleware.cors import CORSMiddleware
22 from starlette.datastructures import State
23
24 import vllm.envs as envs
25 +from vllm.config import VllmConfig
26 from vllm.engine.arg_utils import AsyncEngineArgs
27 from vllm.engine.protocol import EngineClient
28 from vllm.entrypoints.chat_utils import load_chat_template
@@ -198,7 +199,7 @@ def build_app(
199
200 register_sagemaker_api_router(app, supported_tasks)
201
201 - if any(task in supported_tasks for task in ("generate", "render")):
202 + if "generate" in supported_tasks:
203 from vllm.entrypoints.openai.generate.api_router import (
204 register_generate_api_routers,
205 )
@@ -223,6 +224,13 @@ def build_app(
224
225 elastic_ep_attach_router(app)
226
227 + if "generate" in supported_tasks or "render" in supported_tasks:
228 + from vllm.entrypoints.serve.render.api_router import (
229 + attach_router as attach_render_router,
230 + )
231 +
232 + attach_render_router(app)
233 +
234 if "transcription" in supported_tasks:
235 from vllm.entrypoints.openai.speech_to_text.api_router import (
236 attach_router as register_speech_to_text_api_router,
@@ -363,7 +371,7 @@ async def init_app_state(
371 trust_request_chat_template=args.trust_request_chat_template,
372 )
373
366 - if any(task in supported_tasks for task in ("generate", "render")):
374 + if "generate" in supported_tasks:
375 from vllm.entrypoints.openai.generate.api_router import init_generate_state
376
377 await init_generate_state(
@@ -393,6 +401,64 @@ async def init_app_state(
401 state.server_load_metrics = 0
402
403
404 +async def init_render_app_state(
405 + vllm_config: VllmConfig,
406 + state: State,
407 + args: Namespace,
408 +) -> None:
409 + """Initialise FastAPI app state for a CPU-only render server.
410 +
411 + Unlike :func:`init_app_state` this function does not require an
412 + :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
413 + preprocessing pipeline (renderer, io_processor, input_processor)
414 + directly from the :class:`~vllm.config.VllmConfig`.
415 + """
416 + from vllm.entrypoints.chat_utils import load_chat_template
417 + from vllm.entrypoints.serve.render.serving import OpenAIServingRender
418 + from vllm.plugins.io_processors import get_io_processor
419 + from vllm.renderers import renderer_from_config
420 +
421 + served_model_names = args.served_model_name or [args.model]
422 +
423 + if args.enable_log_requests:
424 + request_logger = RequestLogger(max_log_len=args.max_log_len)
425 + else:
426 + request_logger = None
427 +
428 + renderer = renderer_from_config(vllm_config)
429 + io_processor = get_io_processor(
430 + vllm_config, renderer, vllm_config.model_config.io_processor_plugin
431 + )
432 + resolved_chat_template = load_chat_template(args.chat_template)
433 +
434 + state.openai_serving_render = OpenAIServingRender(
435 + model_config=vllm_config.model_config,
436 + renderer=renderer,
437 + io_processor=io_processor,
438 + served_model_names=served_model_names,
439 + request_logger=request_logger,
440 + chat_template=resolved_chat_template,
441 + chat_template_content_format=args.chat_template_content_format,
442 + trust_request_chat_template=args.trust_request_chat_template,
443 + enable_auto_tools=args.enable_auto_tool_choice,
444 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
445 + tool_parser=args.tool_call_parser,
446 + default_chat_template_kwargs=args.default_chat_template_kwargs,
447 + log_error_stack=args.log_error_stack,
448 + )
449 +
450 + # Expose models endpoint via the render handler.
451 + state.openai_serving_models = state.openai_serving_render
452 +
453 + state.vllm_config = vllm_config
454 + # Disable stats logging — there is no engine to poll.
455 + state.log_stats = False
456 + state.engine_client = None
457 + state.args = args
458 + state.enable_server_load_tracking = False
459 + state.server_load_metrics = 0
460 +
461 +
462 def create_server_socket(addr: tuple[str, int]) -> socket.socket:
463 family = socket.AF_INET
464 if is_valid_ipv6_address(addr[0]):
@@ -494,7 +560,6 @@ async def build_and_serve(
560
561 supported_tasks = await engine_client.get_supported_tasks()
562 logger.info("Supported tasks: %s", supported_tasks)
497 -
563 app = build_app(args, supported_tasks)
564 await init_app_state(engine_client, app.state, args, supported_tasks)
565
@@ -522,6 +587,51 @@ async def build_and_serve(
587 )
588
589
590 +async def build_and_serve_renderer(
591 + vllm_config: VllmConfig,
592 + listen_address: str,
593 + sock: socket.socket,
594 + args: Namespace,
595 + **uvicorn_kwargs,
596 +) -> asyncio.Task:
597 + """Build FastAPI app for a CPU-only render server, initialize state, and
598 + start serving.
599 +
600 + Returns the shutdown task for the caller to await.
601 + """
602 +
603 + # Get uvicorn log config (from file or with endpoint filter)
604 + log_config = get_uvicorn_log_config(args)
605 + if log_config is not None:
606 + uvicorn_kwargs["log_config"] = log_config
607 +
608 + app = build_app(args, ("render",))
609 + await init_render_app_state(vllm_config, app.state, args)
610 +
611 + logger.info("Starting vLLM server on %s", listen_address)
612 +
613 + return await serve_http(
614 + app,
615 + sock=sock,
616 + enable_ssl_refresh=args.enable_ssl_refresh,
617 + host=args.host,
618 + port=args.port,
619 + log_level=args.uvicorn_log_level,
620 + # NOTE: When the 'disable_uvicorn_access_log' value is True,
621 + # no access log will be output.
622 + access_log=not args.disable_uvicorn_access_log,
623 + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
624 + ssl_keyfile=args.ssl_keyfile,
625 + ssl_certfile=args.ssl_certfile,
626 + ssl_ca_certs=args.ssl_ca_certs,
627 + ssl_cert_reqs=args.ssl_cert_reqs,
628 + ssl_ciphers=args.ssl_ciphers,
629 + h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
630 + h11_max_header_count=args.h11_max_header_count,
631 + **uvicorn_kwargs,
632 + )
633 +
634 +
635 async def run_server(args, **uvicorn_kwargs) -> None:
636 """Run a single-worker API server."""
637
vllm/entrypoints/openai/chat_completion/api_router.py
-29
@@ -71,34 +71,5 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
71 return StreamingResponse(content=generator, media_type="text/event-stream")
72
73
74 -@router.post(
75 - "/v1/chat/completions/render",
76 - dependencies=[Depends(validate_json_request)],
77 - response_model=list,
78 - responses={
79 - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
80 - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
81 - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
82 - HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
83 - },
84 -)
85 -async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
86 - """Render chat completion request and return conversation and engine
87 - prompts without generating."""
88 - handler = chat(raw_request)
89 - if handler is None:
90 - base_server = raw_request.app.state.openai_serving_tokenization
91 - return base_server.create_error_response(
92 - message="The model does not support Chat Completions API"
93 - )
94 -
95 - result = await handler.render_chat_request(request)
96 -
97 - if isinstance(result, ErrorResponse):
98 - return JSONResponse(content=result.model_dump(), status_code=result.error.code)
99 -
100 - return JSONResponse(content=result)
101 -
102 -
74 def attach_router(app: FastAPI):
75 app.include_router(router)
vllm/entrypoints/openai/completion/api_router.py
-27
@@ -69,32 +69,5 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
69 return StreamingResponse(content=generator, media_type="text/event-stream")
70
71
72 -@router.post(
73 - "/v1/completions/render",
74 - dependencies=[Depends(validate_json_request)],
75 - response_model=list,
76 - responses={
77 - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
78 - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
79 - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
80 - },
81 -)
82 -async def render_completion(request: CompletionRequest, raw_request: Request):
83 - """render completion request and return engine prompts without generating."""
84 - handler = completion(raw_request)
85 - if handler is None:
86 - base_server = raw_request.app.state.openai_serving_tokenization
87 - return base_server.create_error_response(
88 - message="The model does not support Completions API"
89 - )
90 -
91 - result = await handler.render_completion_request(request)
92 -
93 - if isinstance(result, ErrorResponse):
94 - return JSONResponse(content=result.model_dump(), status_code=result.error.code)
95 -
96 - return JSONResponse(content=result)
97 -
98 -
72 def attach_router(app: FastAPI):
73 app.include_router(router)
vllm/entrypoints/openai/generate/api_router.py
+25 -2
@@ -111,7 +111,7 @@ async def init_generate_state(
111 enable_log_outputs=args.enable_log_outputs,
112 enable_log_deltas=args.enable_log_deltas,
113 )
114 - if any(task in supported_tasks for task in ("generate", "render"))
114 + if "generate" in supported_tasks
115 else None
116 )
117 # Warm up chat template processing to avoid first-request latency
@@ -126,7 +126,7 @@ async def init_generate_state(
126 enable_prompt_tokens_details=args.enable_prompt_tokens_details,
127 enable_force_include_usage=args.enable_force_include_usage,
128 )
129 - if any(task in supported_tasks for task in ("generate", "render"))
129 + if "generate" in supported_tasks
130 else None
131 )
132 state.anthropic_serving_messages = (
@@ -160,3 +160,26 @@ async def init_generate_state(
160 if "generate" in supported_tasks
161 else None
162 )
163 +
164 + # Render endpoints are always backed by OpenAIServingRender so that
165 + # /v1/chat/completions/render and /v1/completions/render work on both
166 + # generate-mode and render-only servers.
167 + from vllm.entrypoints.serve.render.serving import OpenAIServingRender
168 +
169 + state.openai_serving_render = OpenAIServingRender(
170 + model_config=engine_client.model_config,
171 + renderer=engine_client.renderer,
172 + io_processor=engine_client.io_processor,
173 + served_model_names=[
174 + mp.name for mp in state.openai_serving_models.base_model_paths
175 + ],
176 + request_logger=request_logger,
177 + chat_template=resolved_chat_template,
178 + chat_template_content_format=args.chat_template_content_format,
179 + trust_request_chat_template=args.trust_request_chat_template,
180 + enable_auto_tools=args.enable_auto_tool_choice,
181 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
182 + tool_parser=args.tool_call_parser,
183 + default_chat_template_kwargs=args.default_chat_template_kwargs,
184 + log_error_stack=args.log_error_stack,
185 + )
vllm/entrypoints/serve/instrumentator/health.py
+5 -1
@@ -22,8 +22,12 @@ def engine_client(request: Request) -> EngineClient:
22 @router.get("/health", response_class=Response)
23 async def health(raw_request: Request) -> Response:
24 """Health check."""
25 + client = engine_client(raw_request)
26 + if client is None:
27 + # Render-only servers have no engine; they are always healthy.
28 + return Response(status_code=200)
29 try:
26 - await engine_client(raw_request).check_health()
30 + await client.check_health()
31 return Response(status_code=200)
32 except EngineDeadError:
33 return Response(status_code=503)
vllm/entrypoints/serve/render/__init__.py new
+2
@@ -0,0 +1,2 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/serve/render/api_router.py new
+87
@@ -0,0 +1,87 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from http import HTTPStatus
4 +
5 +from fastapi import APIRouter, Depends, FastAPI, Request
6 +from fastapi.responses import JSONResponse
7 +
8 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9 +from vllm.entrypoints.openai.completion.protocol import CompletionRequest
10 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse
11 +from vllm.entrypoints.openai.utils import validate_json_request
12 +from vllm.entrypoints.serve.render.serving import OpenAIServingRender
13 +from vllm.entrypoints.utils import create_error_response
14 +from vllm.logger import init_logger
15 +
16 +logger = init_logger(__name__)
17 +
18 +router = APIRouter()
19 +
20 +
21 +def render(request: Request) -> OpenAIServingRender | None:
22 + return getattr(request.app.state, "openai_serving_render", None)
23 +
24 +
25 +@router.post(
26 + "/v1/chat/completions/render",
27 + dependencies=[Depends(validate_json_request)],
28 + response_model=list,
29 + responses={
30 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
31 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
32 + HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
33 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
34 + },
35 +)
36 +async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
37 + handler = render(raw_request)
38 + if handler is None:
39 + error = create_error_response(
40 + message="The model does not support Chat Completions Render API",
41 + err_type="NotFoundError",
42 + status_code=HTTPStatus.NOT_FOUND,
43 + )
44 + return JSONResponse(
45 + status_code=HTTPStatus.NOT_FOUND, content=error.model_dump()
46 + )
47 +
48 + result = await handler.render_chat_request(request)
49 +
50 + if isinstance(result, ErrorResponse):
51 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
52 +
53 + return JSONResponse(content=result)
54 +
55 +
56 +@router.post(
57 + "/v1/completions/render",
58 + dependencies=[Depends(validate_json_request)],
59 + response_model=list,
60 + responses={
61 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
62 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
63 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
64 + },
65 +)
66 +async def render_completion(request: CompletionRequest, raw_request: Request):
67 + handler = render(raw_request)
68 + if handler is None:
69 + error = create_error_response(
70 + message="The model does not support Completions Render API",
71 + err_type="NotFoundError",
72 + status_code=HTTPStatus.NOT_FOUND,
73 + )
74 + return JSONResponse(
75 + status_code=HTTPStatus.NOT_FOUND, content=error.model_dump()
76 + )
77 +
78 + result = await handler.render_completion_request(request)
79 +
80 + if isinstance(result, ErrorResponse):
81 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
82 +
83 + return JSONResponse(content=result)
84 +
85 +
86 +def attach_router(app: FastAPI) -> None:
87 + app.include_router(router)
vllm/entrypoints/serve/render/serving.py new
+475
@@ -0,0 +1,475 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import sys
4 +import traceback
5 +from collections.abc import Callable, Sequence
6 +from http import HTTPStatus
7 +from typing import Any
8 +
9 +import jinja2
10 +from openai_harmony import Message as OpenAIMessage
11 +
12 +from vllm.config import ModelConfig
13 +from vllm.entrypoints.chat_utils import (
14 + ChatTemplateContentFormatOption,
15 + ConversationMessage,
16 +)
17 +from vllm.entrypoints.logger import RequestLogger
18 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
19 +from vllm.entrypoints.openai.completion.protocol import CompletionRequest
20 +from vllm.entrypoints.openai.engine.protocol import (
21 + ErrorInfo,
22 + ErrorResponse,
23 + ModelCard,
24 + ModelList,
25 + ModelPermission,
26 +)
27 +from vllm.entrypoints.openai.parser.harmony_utils import (
28 + get_developer_message,
29 + get_system_message,
30 + parse_chat_inputs_to_harmony_messages,
31 + render_for_completion,
32 +)
33 +from vllm.entrypoints.utils import sanitize_message
34 +from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
35 +from vllm.logger import init_logger
36 +from vllm.parser import ParserManager
37 +from vllm.renderers import BaseRenderer, merge_kwargs
38 +from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
39 +from vllm.tokenizers import TokenizerLike
40 +from vllm.tool_parsers import ToolParser
41 +from vllm.utils.mistral import is_mistral_tokenizer
42 +from vllm.utils.mistral import mt as _mt
43 +
44 +logger = init_logger(__name__)
45 +
46 +
47 +class OpenAIServingRender:
48 + def __init__(
49 + self,
50 + model_config: ModelConfig,
51 + renderer: BaseRenderer,
52 + io_processor: Any,
53 + served_model_names: list[str],
54 + *,
55 + request_logger: RequestLogger | None,
56 + chat_template: str | None,
57 + chat_template_content_format: ChatTemplateContentFormatOption,
58 + trust_request_chat_template: bool = False,
59 + enable_auto_tools: bool = False,
60 + exclude_tools_when_tool_choice_none: bool = False,
61 + tool_parser: str | None = None,
62 + default_chat_template_kwargs: dict[str, Any] | None = None,
63 + log_error_stack: bool = False,
64 + ) -> None:
65 + self.model_config = model_config
66 + self.renderer = renderer
67 + self.io_processor = io_processor
68 + self.served_model_names = served_model_names
69 + self.request_logger = request_logger
70 + self.chat_template = chat_template
71 + self.chat_template_content_format: ChatTemplateContentFormatOption = (
72 + chat_template_content_format
73 + )
74 + self.trust_request_chat_template = trust_request_chat_template
75 + self.enable_auto_tools = enable_auto_tools
76 + self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
77 + self.tool_parser: Callable[[TokenizerLike], ToolParser] | None = (
78 + ParserManager.get_tool_parser(
79 + tool_parser_name=tool_parser,
80 + enable_auto_tools=enable_auto_tools,
81 + model_name=model_config.model,
82 + )
83 + )
84 + self.default_chat_template_kwargs: dict[str, Any] = (
85 + default_chat_template_kwargs or {}
86 + )
87 + self.log_error_stack = log_error_stack
88 + self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
89 + self.supports_browsing = False
90 + self.supports_code_interpreter = False
91 +
92 + async def render_chat_request(
93 + self,
94 + request: ChatCompletionRequest,
95 + ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
96 + """Copied from OpenAIServingChat.render_chat_request.
97 +
98 + Differences: engine_client.errored check removed (no engine client).
99 + """
100 + error_check_ret = await self._check_model(request)
101 + if error_check_ret is not None:
102 + logger.error("Error with model %s", error_check_ret)
103 + return error_check_ret
104 +
105 + try:
106 + tokenizer = self.renderer.tokenizer
107 +
108 + tool_parser = self.tool_parser
109 +
110 + if is_mistral_tokenizer(tokenizer):
111 + # because of issues with pydantic we need to potentially
112 + # re-serialize the tool_calls field of the request
113 + # for more info: see comment in `maybe_serialize_tool_calls`
114 + _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
115 + _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
116 + _mt.validate_request_params(request)
117 +
118 + # Check if tool parsing is unavailable (common condition)
119 + tool_parsing_unavailable = (
120 + tool_parser is None
121 + and not is_mistral_tokenizer(tokenizer)
122 + and not self.use_harmony
123 + )
124 +
125 + # Validate tool_choice when tool parsing is required but unavailable
126 + if tool_parsing_unavailable and request.tool_choice not in (
127 + None,
128 + "none",
129 + ):
130 + if request.tool_choice == "auto" and not self.enable_auto_tools:
131 + # for hf tokenizers, "auto" tools requires
132 + # --enable-auto-tool-choice and --tool-call-parser
133 + return self.create_error_response(
134 + '"auto" tool choice requires '
135 + "--enable-auto-tool-choice and --tool-call-parser to be set"
136 + )
137 + elif request.tool_choice != "auto":
138 + # "required" or named tool requires tool parser
139 + return self.create_error_response(
140 + f'tool_choice="{request.tool_choice}" requires '
141 + "--tool-call-parser to be set"
142 + )
143 +
144 + if request.tools is None or (
145 + request.tool_choice == "none"
146 + and self.exclude_tools_when_tool_choice_none
147 + ):
148 + tool_dicts = None
149 + else:
150 + tool_dicts = [tool.model_dump() for tool in request.tools]
151 +
152 + if not self.use_harmony:
153 + # Common case.
154 + error_check_ret = self._validate_chat_template(
155 + request_chat_template=request.chat_template,
156 + chat_template_kwargs=request.chat_template_kwargs,
157 + trust_request_chat_template=self.trust_request_chat_template,
158 + )
159 + if error_check_ret is not None:
160 + return error_check_ret
161 +
162 + conversation, engine_prompts = await self._preprocess_chat(
163 + request,
164 + request.messages,
165 + default_template=self.chat_template,
166 + default_template_content_format=self.chat_template_content_format,
167 + default_template_kwargs=self.default_chat_template_kwargs,
168 + tool_dicts=tool_dicts,
169 + tool_parser=tool_parser,
170 + )
171 + else:
172 + # For GPT-OSS.
173 + should_include_tools = tool_dicts is not None
174 + conversation, engine_prompts = self._make_request_with_harmony(
175 + request, should_include_tools
176 + )
177 + except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
178 + logger.exception("Error in preprocessing prompt inputs")
179 + return self.create_error_response(e)
180 +
181 + return conversation, engine_prompts
182 +
183 + async def render_completion_request(
184 + self,
185 + request: CompletionRequest,
186 + ) -> list[ProcessorInputs] | ErrorResponse:
187 + """Copied from OpenAIServingCompletion.render_completion_request.
188 +
189 + Differences: engine_client.errored check removed (no engine client).
190 + """
191 + error_check_ret = await self._check_model(request)
192 + if error_check_ret is not None:
193 + return error_check_ret
194 +
195 + # Return error for unsupported features.
196 + if request.suffix is not None:
197 + return self.create_error_response("suffix is not currently supported")
198 +
199 + if request.echo and request.prompt_embeds is not None:
200 + return self.create_error_response("Echo is unsupported with prompt embeds.")
201 +
202 + if request.prompt_logprobs is not None and request.prompt_embeds is not None:
203 + return self.create_error_response(
204 + "prompt_logprobs is not compatible with prompt embeds."
205 + )
206 +
207 + try:
208 + engine_prompts = await self._preprocess_completion(
209 + request,
210 + prompt_input=request.prompt,
211 + prompt_embeds=request.prompt_embeds,
212 + )
213 + except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
214 + logger.exception("Error in preprocessing prompt inputs")
215 + return self.create_error_response(e)
216 +
217 + return engine_prompts
218 +
219 + def _make_request_with_harmony(
220 + self,
221 + request: ChatCompletionRequest,
222 + should_include_tools: bool = True,
223 + ):
224 + """Copied from OpenAIServingChat._make_request_with_harmony."""
225 + messages: list[OpenAIMessage] = []
226 +
227 + # because of issues with pydantic we need to potentially
228 + # re-serialize the tool_calls field of the request
229 + # for more info: see comment in `maybe_serialize_tool_calls`
230 + _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
231 +
232 + # Add system message.
233 + # NOTE: In Chat Completion API, browsing is enabled by default
234 + # if the model supports it. TODO: Support browsing.
235 + assert not self.supports_browsing
236 + assert not self.supports_code_interpreter
237 + sys_msg = get_system_message(
238 + reasoning_effort=request.reasoning_effort,
239 + browser_description=None,
240 + python_description=None,
241 + with_custom_tools=should_include_tools,
242 + )
243 + messages.append(sys_msg)
244 +
245 + # Add developer message.
246 + if request.tools:
247 + dev_msg = get_developer_message(
248 + tools=request.tools if should_include_tools else None # type: ignore[arg-type]
249 + )
250 + messages.append(dev_msg)
251 +
252 + # Add user message.
253 + messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
254 +
255 + # Render prompt token ids.
256 + prompt_token_ids = render_for_completion(messages)
257 + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
258 +
259 + # Add cache_salt if provided in the request
260 + if request.cache_salt is not None:
261 + engine_prompt["cache_salt"] = request.cache_salt
262 +
263 + return messages, [engine_prompt]
264 +
265 + async def show_available_models(self) -> ModelList:
266 + """Returns the models served by this render server."""
267 + max_model_len = self.model_config.max_model_len
268 + return ModelList(
269 + data=[
270 + ModelCard(
271 + id=name,
272 + max_model_len=max_model_len,
273 + root=self.model_config.model,
274 + permission=[ModelPermission()],
275 + )
276 + for name in self.served_model_names
277 + ]
278 + )
279 +
280 + def create_error_response(
281 + self,
282 + message: str | Exception,
283 + err_type: str = "BadRequestError",
284 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
285 + param: str | None = None,
286 + ) -> ErrorResponse:
287 + """Copied from OpenAIServing.create_error_response."""
288 + exc: Exception | None = None
289 +
290 + if isinstance(message, Exception):
291 + exc = message
292 +
293 + from vllm.exceptions import VLLMValidationError
294 +
295 + if isinstance(exc, VLLMValidationError):
296 + err_type = "BadRequestError"
297 + status_code = HTTPStatus.BAD_REQUEST
298 + param = exc.parameter
299 + elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
300 + # Common validation errors from user input
301 + err_type = "BadRequestError"
302 + status_code = HTTPStatus.BAD_REQUEST
303 + param = None
304 + elif isinstance(exc, NotImplementedError):
305 + err_type = "NotImplementedError"
306 + status_code = HTTPStatus.NOT_IMPLEMENTED
307 + param = None
308 + elif exc.__class__.__name__ == "TemplateError":
309 + # jinja2.TemplateError (avoid importing jinja2)
310 + err_type = "BadRequestError"
311 + status_code = HTTPStatus.BAD_REQUEST
312 + param = None
313 + else:
314 + err_type = "InternalServerError"
315 + status_code = HTTPStatus.INTERNAL_SERVER_ERROR
316 + param = None
317 +
318 + message = str(exc)
319 +
320 + if self.log_error_stack:
321 + exc_type, _, _ = sys.exc_info()
322 + if exc_type is not None:
323 + traceback.print_exc()
324 + else:
325 + traceback.print_stack()
326 +
327 + return ErrorResponse(
328 + error=ErrorInfo(
329 + message=sanitize_message(message),
330 + type=err_type,
331 + code=status_code.value,
332 + param=param,
333 + )
334 + )
335 +
336 + def _is_model_supported(self, model_name: str) -> bool:
337 + """Simplified from OpenAIServing._is_model_supported (no LoRA support)."""
338 + return model_name in self.served_model_names
339 +
340 + async def _check_model(
341 + self,
342 + request: Any,
343 + ) -> ErrorResponse | None:
344 + """Simplified from OpenAIServing._check_model (no LoRA support)."""
345 + if self._is_model_supported(request.model):
346 + return None
347 + return self.create_error_response(
348 + message=f"The model `{request.model}` does not exist.",
349 + err_type="NotFoundError",
350 + status_code=HTTPStatus.NOT_FOUND,
351 + param="model",
352 + )
353 +
354 + def _validate_chat_template(
355 + self,
356 + request_chat_template: str | None,
357 + chat_template_kwargs: dict[str, Any] | None,
358 + trust_request_chat_template: bool,
359 + ) -> ErrorResponse | None:
360 + """Copied from OpenAIServing._validate_chat_template."""
361 + if not trust_request_chat_template and (
362 + request_chat_template is not None
363 + or (
364 + chat_template_kwargs
365 + and chat_template_kwargs.get("chat_template") is not None
366 + )
367 + ):
368 + return self.create_error_response(
369 + "Chat template is passed with request, but "
370 + "--trust-request-chat-template is not set. "
371 + "Refused request with untrusted chat template."
372 + )
373 + return None
374 +
375 + async def _preprocess_completion(
376 + self,
377 + request: Any,
378 + prompt_input: str | list[str] | list[int] | list[list[int]] | None,
379 + prompt_embeds: bytes | list[bytes] | None,
380 + ) -> list[ProcessorInputs]:
381 + """Copied from OpenAIServing._preprocess_completion."""
382 + prompts = list[SingletonPrompt | bytes]()
383 + if prompt_embeds is not None: # embeds take higher priority
384 + prompts.extend(prompt_to_seq(prompt_embeds))
385 + if prompt_input is not None:
386 + prompts.extend(prompt_to_seq(prompt_input))
387 + return await self._preprocess_cmpl(request, prompts)
388 +
389 + async def _preprocess_cmpl(
390 + self,
391 + request: Any,
392 + prompts: Sequence[PromptType | bytes],
393 + ) -> list[ProcessorInputs]:
394 + """Copied from OpenAIServing._preprocess_cmpl."""
395 + renderer = self.renderer
396 + model_config = self.model_config
397 +
398 + parsed_prompts = [
399 + (
400 + prompt
401 + if isinstance(prompt, bytes)
402 + else parse_model_prompt(model_config, prompt)
403 + )
404 + for prompt in prompts
405 + ]
406 + tok_params = request.build_tok_params(model_config)
407 +
408 + return await renderer.render_cmpl_async(
409 + parsed_prompts,
410 + tok_params,
411 + prompt_extras={
412 + k: v
413 + for k in ("mm_processor_kwargs", "cache_salt")
414 + if (v := getattr(request, k, None)) is not None
415 + },
416 + )
417 +
418 + async def _preprocess_chat(
419 + self,
420 + request: Any,
421 + messages: list[Any],
422 + default_template: str | None,
423 + default_template_content_format: ChatTemplateContentFormatOption,
424 + default_template_kwargs: dict[str, Any] | None,
425 + tool_dicts: list[dict[str, Any]] | None = None,
426 + tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
427 + ) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
428 + """Copied from OpenAIServing._preprocess_chat.
429 +
430 + Differences: isinstance check is ChatCompletionRequest-only
431 + (ResponsesRequest not supported here); TODO comment dropped accordingly.
432 + """
433 + renderer = self.renderer
434 +
435 + default_template_kwargs = merge_kwargs(
436 + default_template_kwargs,
437 + dict(
438 + tools=tool_dicts,
439 + tokenize=is_mistral_tokenizer(renderer.tokenizer),
440 + ),
441 + )
442 +
443 + tok_params = request.build_tok_params(self.model_config)
444 + chat_params = request.build_chat_params(
445 + default_template, default_template_content_format
446 + ).with_defaults(default_template_kwargs)
447 +
448 + (conversation,), (engine_prompt,) = await renderer.render_chat_async(
449 + [messages],
450 + chat_params,
451 + tok_params,
452 + prompt_extras={
453 + k: v
454 + for k in ("mm_processor_kwargs", "cache_salt")
455 + if (v := getattr(request, k, None)) is not None
456 + },
457 + )
458 +
459 + # tool parsing is done only if a tool_parser has been set and if
460 + # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
461 + # is set, we want to prevent parsing a tool_call hallucinated by the LLM
462 + if tool_parser is not None:
463 + tool_choice = getattr(request, "tool_choice", "none")
464 + if tool_choice != "none":
465 + if not isinstance(request, ChatCompletionRequest):
466 + msg = (
467 + "Tool usage is only supported "
468 + " for ChatCompletionRequest, but got "
469 + f"{type(request).__name__}"
470 + )
471 + raise NotImplementedError(msg)
472 + tokenizer = renderer.get_tokenizer()
473 + request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]
474 +
475 + return conversation, [engine_prompt]
vllm/v1/engine/launch.py deleted
-204
@@ -1,204 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""
4 -LaunchEngineClient: A lightweight EngineClient for GPU-less online serving.
5 -
6 -This implements the EngineClient protocol without AsyncLLM or EngineCore,
7 -enabling preprocessing (tokenization, rendering) and postprocessing
8 -(detokenization) without GPU inference.
9 -"""
10 -
11 -from collections.abc import AsyncGenerator, Iterable, Mapping
12 -from typing import Any
13 -
14 -from vllm.config import VllmConfig
15 -from vllm.engine.protocol import EngineClient, StreamingInput
16 -from vllm.inputs import ProcessorInputs, PromptType
17 -from vllm.logger import init_logger
18 -from vllm.lora.request import LoRARequest
19 -from vllm.outputs import PoolingRequestOutput, RequestOutput
20 -from vllm.plugins.io_processors import get_io_processor
21 -from vllm.pooling_params import PoolingParams
22 -from vllm.renderers import renderer_from_config
23 -from vllm.sampling_params import SamplingParams
24 -from vllm.tasks import SupportedTask
25 -from vllm.v1.engine import EngineCoreRequest, PauseMode
26 -from vllm.v1.engine.input_processor import InputProcessor
27 -
28 -logger = init_logger(__name__)
29 -
30 -
31 -class LaunchEngineClient(EngineClient):
32 - """GPU-less EngineClient that only supports preprocessing/postprocessing.
33 -
34 - This is a Null Object at the EngineClient level, bypassing AsyncLLM
35 - entirely. It initializes renderer, io_processor, and input_processor
36 - for tokenization and rendering, but raises NotImplementedError for
37 - any inference-related operations.
38 - """
39 -
40 - def __init__(
41 - self,
42 - vllm_config: VllmConfig,
43 - ) -> None:
44 - self.vllm_config = vllm_config
45 - self.model_config = vllm_config.model_config
46 -
47 - self.renderer = renderer = renderer_from_config(self.vllm_config)
48 - self.io_processor = get_io_processor(
49 - self.vllm_config,
50 - self.renderer,
51 - self.model_config.io_processor_plugin,
52 - )
53 -
54 - # Convert TokPrompt --> EngineCoreRequest.
55 - self.input_processor = InputProcessor(self.vllm_config, renderer)
56 -
57 - @classmethod
58 - def from_vllm_config(
59 - cls,
60 - vllm_config: VllmConfig,
61 - ) -> "LaunchEngineClient":
62 - """Create a LaunchEngineClient from a VllmConfig without GPU."""
63 - return cls(
64 - vllm_config=vllm_config,
65 - )
66 -
67 - # -- Task support --
68 -
69 - async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
70 - return ("render",)
71 -
72 - # -- Inference (not supported) --
73 -
74 - async def generate(
75 - self,
76 - prompt: EngineCoreRequest
77 - | PromptType
78 - | ProcessorInputs
79 - | AsyncGenerator[StreamingInput, None],
80 - sampling_params: SamplingParams,
81 - request_id: str,
82 - *,
83 - prompt_text: str | None = None,
84 - lora_request: LoRARequest | None = None,
85 - tokenization_kwargs: dict[str, Any] | None = None,
86 - trace_headers: Mapping[str, str] | None = None,
87 - priority: int = 0,
88 - data_parallel_rank: int | None = None,
89 - reasoning_ended: bool | None = None,
90 - ) -> AsyncGenerator[RequestOutput, None]:
91 - raise NotImplementedError(
92 - "LaunchEngineClient does not support inference. "
93 - "Use vllm serve for generation requests."
94 - )
95 - # yield is needed to make this an async generator
96 - yield # type: ignore[misc] # pragma: no cover
97 -
98 - # -- Request management (no-op) --
99 -
100 - async def abort(
101 - self, request_id: str | Iterable[str], internal: bool = False
102 - ) -> None:
103 - pass
104 -
105 - # -- Generation control (no-op) --
106 -
107 - async def pause_generation(
108 - self,
109 - *,
110 - mode: PauseMode = "abort",
111 - wait_for_inflight_requests: bool | None = None,
112 - clear_cache: bool = True,
113 - ) -> None:
114 - pass
115 -
116 - async def resume_generation(self) -> None:
117 - pass
118 -
119 - async def is_paused(self) -> bool:
120 - return False
121 -
122 - def shutdown(self, timeout: float | None = None) -> None:
123 - pass
124 -
125 - async def encode(
126 - self,
127 - prompt: PromptType | ProcessorInputs,
128 - pooling_params: PoolingParams,
129 - request_id: str,
130 - lora_request: LoRARequest | None = None,
131 - trace_headers: Mapping[str, str] | None = None,
132 - priority: int = 0,
133 - tokenization_kwargs: dict[str, Any] | None = None,
134 - reasoning_ended: bool | None = None,
135 - ) -> AsyncGenerator[PoolingRequestOutput, None]:
136 - raise NotImplementedError(
137 - "LaunchEngineClient does not support inference. "
138 - "Use vllm serve for encoding requests."
139 - )
140 - yield # type: ignore[misc] # pragma: no cover
141 -
142 - # -- Observability (no-op / defaults) --
143 -
144 - async def is_tracing_enabled(self) -> bool:
145 - return False
146 -
147 - async def do_log_stats(self) -> None:
148 - pass
149 -
150 - async def check_health(self) -> None:
151 - pass
152 -
153 - async def start_profile(self) -> None:
154 - pass
155 -
156 - async def stop_profile(self) -> None:
157 - pass
158 -
159 - # -- Cache management (no-op) --
160 -
161 - async def reset_mm_cache(self) -> None:
162 - pass
163 -
164 - async def reset_prefix_cache(
165 - self, reset_running_requests: bool = False, reset_connector: bool = False
166 - ) -> bool:
167 - return True
168 -
169 - async def reset_encoder_cache(self) -> None:
170 - pass
171 -
172 - # -- Power management (no-op) --
173 -
174 - async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
175 - pass
176 -
177 - async def wake_up(self, tags: list[str] | None = None) -> None:
178 - pass
179 -
180 - async def is_sleeping(self) -> bool:
181 - return False
182 -
183 - # -- LoRA (not supported) --
184 -
185 - async def add_lora(self, lora_request: LoRARequest) -> bool:
186 - return False
187 -
188 - # -- Status properties --
189 -
190 - @property
191 - def is_running(self) -> bool:
192 - return True
193 -
194 - @property
195 - def is_stopped(self) -> bool:
196 - return False
197 -
198 - @property
199 - def errored(self) -> bool:
200 - return False
201 -
202 - @property
203 - def dead_error(self) -> BaseException:
204 - return RuntimeError("LaunchEngineClient does not support inference")