@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc6be89d16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 06:30 UTC

[Frontend] Add vllm launch command for GPU-less preprocessing serving (#34551)

Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>

Hyunkyun Moon committed Mar 5, 2026 at 03:41 UTC bc6be89d16c6a0b3763a3fdc2623b90a9f7da8f1
10 files changed +776 -65
tests/entrypoints/openai/cpu/test_render.py renamed
+1 -1
@@ -7,7 +7,7 @@ import httpx
7 import pytest
8 import pytest_asyncio
9
10 -from ...utils import RemoteOpenAIServer
10 +from tests.utils import RemoteOpenAIServer
11
12 MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
13
tests/entrypoints/openai/test_launch_render.py new
+199
@@ -0,0 +1,199 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""E2E tests for render endpoints via `vllm launch` (GPU-less serving)."""
4 +
5 +import httpx
6 +import pytest
7 +import pytest_asyncio
8 +
9 +from ...utils import RemoteLaunchRenderServer
10 +
11 +MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
12 +
13 +
14 +@pytest.fixture(scope="module")
15 +def server():
16 + args: list[str] = []
17 + with RemoteLaunchRenderServer(MODEL_NAME, args, max_wait_seconds=120) as srv:
18 + yield srv
19 +
20 +
21 +@pytest_asyncio.fixture
22 +async def client(server):
23 + async with httpx.AsyncClient(
24 + base_url=server.url_for(""), timeout=30.0
25 + ) as http_client:
26 + yield http_client
27 +
28 +
29 +# -- Chat Completion Render --
30 +
31 +
32 +@pytest.mark.asyncio
33 +async def test_chat_render_basic(client):
34 + response = await client.post(
35 + "/v1/chat/completions/render",
36 + json={
37 + "model": MODEL_NAME,
38 + "messages": [{"role": "user", "content": "Hello, how are you?"}],
39 + },
40 + )
41 +
42 + assert response.status_code == 200
43 + data = response.json()
44 +
45 + assert isinstance(data, list)
46 + assert len(data) == 2
47 +
48 + conversation, engine_prompts = data
49 +
50 + assert isinstance(conversation, list)
51 + assert conversation[0]["role"] == "user"
52 +
53 + assert isinstance(engine_prompts, list)
54 + assert len(engine_prompts) > 0
55 + first_prompt = engine_prompts[0]
56 + assert "prompt_token_ids" in first_prompt
57 + assert "prompt" in first_prompt
58 + assert isinstance(first_prompt["prompt_token_ids"], list)
59 + assert all(isinstance(t, int) for t in first_prompt["prompt_token_ids"])
60 +
61 +
62 +@pytest.mark.asyncio
63 +async def test_chat_render_multi_turn(client):
64 + response = await client.post(
65 + "/v1/chat/completions/render",
66 + json={
67 + "model": MODEL_NAME,
68 + "messages": [
69 + {"role": "user", "content": "Hello"},
70 + {"role": "assistant", "content": "Hi there!"},
71 + {"role": "user", "content": "How are you?"},
72 + ],
73 + },
74 + )
75 +
76 + assert response.status_code == 200
77 + conversation, engine_prompts = response.json()
78 +
79 + assert len(conversation) == 3
80 + assert conversation[0]["role"] == "user"
81 + assert conversation[1]["role"] == "assistant"
82 + assert conversation[2]["role"] == "user"
83 + assert len(engine_prompts) > 0
84 + assert len(engine_prompts[0]["prompt_token_ids"]) > 0
85 +
86 +
87 +@pytest.mark.asyncio
88 +async def test_chat_render_invalid_model(client):
89 + response = await client.post(
90 + "/v1/chat/completions/render",
91 + json={
92 + "model": "nonexistent-model",
93 + "messages": [{"role": "user", "content": "Hello"}],
94 + },
95 + )
96 +
97 + assert response.status_code == 404
98 + assert "error" in response.json()
99 +
100 +
101 +# -- Completion Render --
102 +
103 +
104 +@pytest.mark.asyncio
105 +async def test_completion_render_basic(client):
106 + response = await client.post(
107 + "/v1/completions/render",
108 + json={
109 + "model": MODEL_NAME,
110 + "prompt": "Once upon a time",
111 + },
112 + )
113 +
114 + assert response.status_code == 200
115 + data = response.json()
116 +
117 + assert isinstance(data, list)
118 + assert len(data) > 0
119 +
120 + first_prompt = data[0]
121 + assert "prompt_token_ids" in first_prompt
122 + assert "prompt" in first_prompt
123 + assert isinstance(first_prompt["prompt_token_ids"], list)
124 + assert len(first_prompt["prompt_token_ids"]) > 0
125 + assert "Once upon a time" in first_prompt["prompt"]
126 +
127 +
128 +@pytest.mark.asyncio
129 +async def test_completion_render_multiple_prompts(client):
130 + response = await client.post(
131 + "/v1/completions/render",
132 + json={
133 + "model": MODEL_NAME,
134 + "prompt": ["Hello world", "Goodbye world"],
135 + },
136 + )
137 +
138 + assert response.status_code == 200
139 + data = response.json()
140 +
141 + assert isinstance(data, list)
142 + assert len(data) == 2
143 +
144 + for prompt in data:
145 + assert "prompt_token_ids" in prompt
146 + assert "prompt" in prompt
147 + assert len(prompt["prompt_token_ids"]) > 0
148 +
149 +
150 +@pytest.mark.asyncio
151 +async def test_completion_render_invalid_model(client):
152 + response = await client.post(
153 + "/v1/completions/render",
154 + json={
155 + "model": "nonexistent-model",
156 + "prompt": "Hello",
157 + },
158 + )
159 +
160 + assert response.status_code == 404
161 + assert "error" in response.json()
162 +
163 +
164 +@pytest.mark.asyncio
165 +async def test_render_is_fast(client):
166 + """Render should complete quickly since there is no inference."""
167 + import time
168 +
169 + start = time.perf_counter()
170 + response = await client.post(
171 + "/v1/completions/render",
172 + json={
173 + "model": MODEL_NAME,
174 + "prompt": "Tell me a very long story about " * 10,
175 + },
176 + )
177 + elapsed = time.perf_counter() - start
178 +
179 + assert response.status_code == 200
180 + assert elapsed < 2.0
181 +
182 +
183 +# -- Health & Models --
184 +
185 +
186 +@pytest.mark.asyncio
187 +async def test_health_endpoint(client):
188 + response = await client.get("/health")
189 + assert response.status_code == 200
190 +
191 +
192 +@pytest.mark.asyncio
193 +async def test_models_endpoint(client):
194 + response = await client.get("/v1/models")
195 + assert response.status_code == 200
196 + data = response.json()
197 + assert "data" in data
198 + model_ids = [m["id"] for m in data["data"]]
199 + assert MODEL_NAME in model_ids
tests/entrypoints/test_launch_cli.py new
+111
@@ -0,0 +1,111 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Unit tests for the `vllm launch` CLI subcommand."""
4 +
5 +import argparse
6 +from unittest.mock import patch
7 +
8 +import pytest
9 +
10 +from vllm.entrypoints.cli.launch import (
11 + LaunchSubcommand,
12 + RenderSubcommand,
13 + cmd_init,
14 +)
15 +from vllm.utils.argparse_utils import FlexibleArgumentParser
16 +
17 +
18 +@pytest.fixture
19 +def launch_parser():
20 + parser = FlexibleArgumentParser(description="test")
21 + subparsers = parser.add_subparsers(required=False, dest="subparser")
22 + LaunchSubcommand().subparser_init(subparsers)
23 + return parser
24 +
25 +
26 +def test_subcommand_name():
27 + assert LaunchSubcommand().name == "launch"
28 +
29 +
30 +def test_cmd_init_returns_subcommand():
31 + result = cmd_init()
32 + assert len(result) == 1
33 + assert isinstance(result[0], LaunchSubcommand)
34 +
35 +
36 +# -- Parsing: `vllm launch render` --
37 +
38 +
39 +def test_parse_launch_render(launch_parser):
40 + args = launch_parser.parse_args(["launch", "render", "--model", "test-model"])
41 + assert args.launch_component == "render"
42 +
43 +
44 +def test_parse_launch_requires_component(launch_parser):
45 + with pytest.raises(SystemExit):
46 + launch_parser.parse_args(["launch", "--model", "test-model"])
47 +
48 +
49 +def test_parse_launch_invalid_component(launch_parser):
50 + with pytest.raises(SystemExit):
51 + launch_parser.parse_args(["launch", "unknown", "--model", "test-model"])
52 +
53 +
54 +# -- Dispatch --
55 +
56 +
57 +def test_cmd_launch_render_calls_run():
58 + args = argparse.Namespace(model_tag=None, model="test-model")
59 + with patch("vllm.entrypoints.cli.launch.uvloop.run") as mock_uvloop_run:
60 + RenderSubcommand.cmd(args)
61 + mock_uvloop_run.assert_called_once()
62 +
63 +
64 +def test_cmd_launch_model_tag_overrides():
65 + args = argparse.Namespace(
66 + model_tag="tag-model",
67 + model="original-model",
68 + launch_command=lambda a: None,
69 + )
70 + LaunchSubcommand.cmd(args)
71 + assert args.model == "tag-model"
72 +
73 +
74 +def test_cmd_launch_model_tag_none():
75 + args = argparse.Namespace(
76 + model_tag=None,
77 + model="original-model",
78 + launch_command=lambda a: None,
79 + )
80 + LaunchSubcommand.cmd(args)
81 + assert args.model == "original-model"
82 +
83 +
84 +def test_cmd_dispatches():
85 + called = {}
86 +
87 + def fake_dispatch(args):
88 + called["args"] = args
89 +
90 + args = argparse.Namespace(launch_command=fake_dispatch)
91 + LaunchSubcommand.cmd(args)
92 + assert "args" in called
93 +
94 +
95 +# -- Module registration --
96 +
97 +
98 +def test_subparser_init_returns_parser():
99 + parser = FlexibleArgumentParser(description="test")
100 + subparsers = parser.add_subparsers(required=False, dest="subparser")
101 + result = LaunchSubcommand().subparser_init(subparsers)
102 + assert isinstance(result, FlexibleArgumentParser)
103 +
104 +
105 +def test_launch_registered_in_main():
106 + """Verify that launch module is importable as a CLI module."""
107 + import vllm.entrypoints.cli.launch as launch_module
108 +
109 + assert hasattr(launch_module, "cmd_init")
110 + subcmds = launch_module.cmd_init()
111 + assert any(s.name == "launch" for s in subcmds)
tests/utils.py
+76 -23
@@ -110,31 +110,25 @@ VLLM_PATH = Path(__file__).parent.parent
110 """Path to root of the vLLM repository."""
111
112
113 -class RemoteOpenAIServer:
113 +class RemoteVLLMServer:
114 + """Base class for launching vLLM server subprocesses for testing.
115 +
116 + Subclasses must override ``_create_cli_subcommand`` and
117 + ``_start_server``.
118 + """
119 +
120 DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
121 + proc: subprocess.Popen
122 +
123 + def _create_cli_subcommand(self):
124 + """Return a CLISubcommand instance used to parse CLI args."""
125 + raise NotImplementedError
126
127 def _start_server(
128 self, model: str, vllm_serve_args: list[str], env_dict: dict[str, str] | None
129 ) -> None:
130 """Subclasses override this method to customize server process launch"""
120 - env = os.environ.copy()
121 - # the current process might initialize cuda,
122 - # to be safe, we should use spawn method
123 - env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
124 - if env_dict is not None:
125 - env.update(env_dict)
126 - serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
127 - print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
128 - print(f"Environment variables: {env}")
129 - self.proc: subprocess.Popen = subprocess.Popen(
130 - serve_cmd,
131 - env=env,
132 - stdout=sys.stdout,
133 - stderr=sys.stderr,
134 - # Create a dedicated process group so we can kill
135 - # the entire tree (parent + EngineCore + workers) at once.
136 - start_new_session=True,
137 - )
131 + raise NotImplementedError
132
133 def __init__(
134 self,
@@ -171,9 +165,9 @@ class RemoteOpenAIServer:
165 json.dumps(override_hf_configs),
166 ]
167
174 - parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
168 + parser = FlexibleArgumentParser(description="vLLM's remote server.")
169 subparsers = parser.add_subparsers(required=False, dest="subparser")
176 - parser = ServeSubcommand().subparser_init(subparsers)
170 + parser = self._create_cli_subcommand().subparser_init(subparsers)
171 args = parser.parse_args(["--model", model, *vllm_serve_args])
172 self.uds = args.uds
173 if args.uds:
@@ -183,7 +177,9 @@ class RemoteOpenAIServer:
177 self.host = str(args.host or "127.0.0.1")
178 self.port = int(args.port)
179
186 - self.show_hidden_metrics = args.show_hidden_metrics_for_version is not None
180 + self.show_hidden_metrics = (
181 + getattr(args, "show_hidden_metrics_for_version", None) is not None
182 + )
183
184 # download the model before starting the server to avoid timeout
185 is_local = os.path.isdir(model)
@@ -201,7 +197,8 @@ class RemoteOpenAIServer:
197 if self._pre_server_gpu_memory is not None:
198 pre_gb = self._pre_server_gpu_memory / 1e9
199 print(
204 - f"[RemoteOpenAIServer] GPU memory before server start: {pre_gb:.2f} GB"
200 + f"[{type(self).__name__}] GPU memory before server start: "
201 + f"{pre_gb:.2f} GB"
202 )
203
204 self._start_server(model, vllm_serve_args, env_dict)
@@ -452,6 +449,62 @@ class RemoteOpenAIServer:
449 )
450
451
452 +class RemoteOpenAIServer(RemoteVLLMServer):
453 + """Launches ``vllm serve`` for testing OpenAI-compatible endpoints."""
454 +
455 + def _create_cli_subcommand(self):
456 + return ServeSubcommand()
457 +
458 + def _start_server(
459 + self, model: str, vllm_serve_args: list[str], env_dict: dict[str, str] | None
460 + ) -> None:
461 + env = os.environ.copy()
462 + # the current process might initialize cuda,
463 + # to be safe, we should use spawn method
464 + env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
465 + if env_dict is not None:
466 + env.update(env_dict)
467 + serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
468 + print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
469 + print(f"Environment variables: {env}")
470 + self.proc: subprocess.Popen = subprocess.Popen(
471 + serve_cmd,
472 + env=env,
473 + stdout=sys.stdout,
474 + stderr=sys.stderr,
475 + # Create a dedicated process group so we can kill
476 + # the entire tree (parent + EngineCore + workers) at once.
477 + start_new_session=True,
478 + )
479 +
480 +
481 +class RemoteLaunchRenderServer(RemoteVLLMServer):
482 + """Launches ``vllm launch render`` for GPU-less serving tests."""
483 +
484 + def _create_cli_subcommand(self):
485 + return ServeSubcommand()
486 +
487 + def _start_server(
488 + self, model: str, vllm_serve_args: list[str], env_dict: dict[str, str] | None
489 + ) -> None:
490 + env = os.environ.copy()
491 + env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
492 + if env_dict is not None:
493 + env.update(env_dict)
494 + serve_cmd = ["vllm", "launch", "render", model, *vllm_serve_args]
495 + print(f"Launching RemoteLaunchRenderServer with: {' '.join(serve_cmd)}")
496 + self.proc: subprocess.Popen = subprocess.Popen(
497 + serve_cmd,
498 + env=env,
499 + stdout=sys.stdout,
500 + stderr=sys.stderr,
501 + start_new_session=True,
502 + )
503 +
504 + def _wait_for_gpu_memory_release(self, timeout: float = 30.0):
505 + pass # No GPU used
506 +
507 +
508 class RemoteOpenAIServerCustom(RemoteOpenAIServer):
509 """Launch test server with custom child process"""
510
vllm/entrypoints/cli/launch.py new
+128
@@ -0,0 +1,128 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import argparse
5 +
6 +import uvloop
7 +
8 +from vllm.engine.arg_utils import AsyncEngineArgs
9 +from vllm.entrypoints.cli.types import CLISubcommand
10 +from vllm.entrypoints.openai.api_server import (
11 + build_and_serve,
12 + setup_server,
13 +)
14 +from vllm.entrypoints.openai.cli_args import (
15 + make_arg_parser,
16 + validate_parsed_serve_args,
17 +)
18 +from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
19 +from vllm.logger import init_logger
20 +from vllm.utils.argparse_utils import FlexibleArgumentParser
21 +
22 +logger = init_logger(__name__)
23 +
24 +DESCRIPTION = "Launch individual vLLM components."
25 +
26 +
27 +class LaunchSubcommandBase(CLISubcommand):
28 + """The base class of subcommands for `vllm launch`."""
29 +
30 + help: str
31 +
32 + @classmethod
33 + def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
34 + """Add the CLI arguments to the parser.
35 +
36 + By default, adds the standard vLLM serving arguments.
37 + Subclasses can override to add component-specific arguments.
38 + """
39 + make_arg_parser(parser)
40 +
41 + @staticmethod
42 + def cmd(args: argparse.Namespace) -> None:
43 + raise NotImplementedError
44 +
45 +
46 +class RenderSubcommand(LaunchSubcommandBase):
47 + """The `render` subcommand for `vllm launch`."""
48 +
49 + name = "render"
50 + help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
51 +
52 + @staticmethod
53 + def cmd(args: argparse.Namespace) -> None:
54 + uvloop.run(run_launch_fastapi(args))
55 +
56 +
57 +class LaunchSubcommand(CLISubcommand):
58 + """The `launch` subcommand for the vLLM CLI.
59 +
60 + Uses nested sub-subcommands so each component can define its own
61 + arguments independently (e.g. ``vllm launch render``).
62 + """
63 +
64 + name = "launch"
65 +
66 + @staticmethod
67 + def cmd(args: argparse.Namespace) -> None:
68 + if hasattr(args, "model_tag") and args.model_tag is not None:
69 + args.model = args.model_tag
70 +
71 + args.launch_command(args)
72 +
73 + def validate(self, args: argparse.Namespace) -> None:
74 + validate_parsed_serve_args(args)
75 +
76 + def subparser_init(
77 + self, subparsers: argparse._SubParsersAction
78 + ) -> FlexibleArgumentParser:
79 + launch_parser = subparsers.add_parser(
80 + self.name,
81 + help=DESCRIPTION,
82 + description=DESCRIPTION,
83 + usage=f"vllm {self.name} <component> [options]",
84 + )
85 + launch_subparsers = launch_parser.add_subparsers(
86 + required=True, dest="launch_component"
87 + )
88 +
89 + for cmd_cls in LaunchSubcommandBase.__subclasses__():
90 + cmd_subparser = launch_subparsers.add_parser(
91 + cmd_cls.name,
92 + help=cmd_cls.help,
93 + description=cmd_cls.help,
94 + usage=f"vllm {self.name} {cmd_cls.name} [options]",
95 + )
96 + cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
97 + cmd_cls.add_cli_args(cmd_subparser)
98 + cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
99 + subcmd=f"{self.name} {cmd_cls.name}"
100 + )
101 +
102 + return launch_parser
103 +
104 +
105 +def cmd_init() -> list[CLISubcommand]:
106 + return [LaunchSubcommand()]
107 +
108 +
109 +async def run_launch_fastapi(args: argparse.Namespace) -> None:
110 + """Run the online serving layer with FastAPI (no GPU inference)."""
111 + from vllm.config import VllmConfig
112 + from vllm.v1.engine.launch import LaunchEngineClient
113 +
114 + # 1. Socket binding
115 + listen_address, sock = setup_server(args)
116 +
117 + # 2. Create LaunchEngineClient (no GPU)
118 + engine_args = AsyncEngineArgs.from_cli_args(args)
119 + model_config = engine_args.create_model_config()
120 + vllm_config = VllmConfig(model_config=model_config)
121 + engine_client = LaunchEngineClient.from_vllm_config(vllm_config)
122 +
123 + # 3. Build app, initialize state, and start serving
124 + shutdown_task = await build_and_serve(engine_client, listen_address, sock, args)
125 + try:
126 + await shutdown_task
127 + finally:
128 + sock.close()
vllm/entrypoints/cli/main.py
+2
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16 def main():
17 import vllm.entrypoints.cli.benchmark.main
18 import vllm.entrypoints.cli.collect_env
19 + import vllm.entrypoints.cli.launch
20 import vllm.entrypoints.cli.openai
21 import vllm.entrypoints.cli.run_batch
22 import vllm.entrypoints.cli.serve
@@ -25,6 +26,7 @@ def main():
26 CMD_MODULES = [
27 vllm.entrypoints.cli.openai,
28 vllm.entrypoints.cli.serve,
29 + vllm.entrypoints.cli.launch,
30 vllm.entrypoints.cli.benchmark.main,
31 vllm.entrypoints.cli.collect_env,
32 vllm.entrypoints.cli.run_batch,
vllm/entrypoints/openai/api_server.py
+52 -38
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import asyncio
4 import importlib
5 import inspect
6 import multiprocessing
@@ -194,7 +195,7 @@ def build_app(
195
196 register_sagemaker_api_router(app, supported_tasks)
197
197 - if "generate" in supported_tasks:
198 + if any(task in supported_tasks for task in ("generate", "render")):
199 from vllm.entrypoints.openai.generate.api_router import (
200 register_generate_api_routers,
201 )
@@ -357,7 +358,7 @@ async def init_app_state(
358 log_error_stack=args.log_error_stack,
359 )
360
360 - if "generate" in supported_tasks:
361 + if any(task in supported_tasks for task in ("generate", "render")):
362 from vllm.entrypoints.openai.generate.api_router import init_generate_state
363
364 await init_generate_state(
@@ -469,6 +470,53 @@ def setup_server(args):
470 return listen_address, sock
471
472
473 +async def build_and_serve(
474 + engine_client: EngineClient,
475 + listen_address: str,
476 + sock: socket.socket,
477 + args: Namespace,
478 + **uvicorn_kwargs,
479 +) -> asyncio.Task:
480 + """Build FastAPI app, initialize state, and start serving.
481 +
482 + Returns the shutdown task for the caller to await.
483 + """
484 +
485 + # Get uvicorn log config (from file or with endpoint filter)
486 + log_config = get_uvicorn_log_config(args)
487 + if log_config is not None:
488 + uvicorn_kwargs["log_config"] = log_config
489 +
490 + supported_tasks = await engine_client.get_supported_tasks()
491 + logger.info("Supported tasks: %s", supported_tasks)
492 +
493 + app = build_app(args, supported_tasks)
494 + await init_app_state(engine_client, app.state, args, supported_tasks)
495 +
496 + logger.info("Starting vLLM server on %s", listen_address)
497 +
498 + return await serve_http(
499 + app,
500 + sock=sock,
501 + enable_ssl_refresh=args.enable_ssl_refresh,
502 + host=args.host,
503 + port=args.port,
504 + log_level=args.uvicorn_log_level,
505 + # NOTE: When the 'disable_uvicorn_access_log' value is True,
506 + # no access log will be output.
507 + access_log=not args.disable_uvicorn_access_log,
508 + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
509 + ssl_keyfile=args.ssl_keyfile,
510 + ssl_certfile=args.ssl_certfile,
511 + ssl_ca_certs=args.ssl_ca_certs,
512 + ssl_cert_reqs=args.ssl_cert_reqs,
513 + ssl_ciphers=args.ssl_ciphers,
514 + h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
515 + h11_max_header_count=args.h11_max_header_count,
516 + **uvicorn_kwargs,
517 + )
518 +
519 +
520 async def run_server(args, **uvicorn_kwargs) -> None:
521 """Run a single-worker API server."""
522
@@ -490,47 +538,13 @@ async def run_server_worker(
538 if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
539 ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
540
493 - # Get uvicorn log config (from file or with endpoint filter)
494 - log_config = get_uvicorn_log_config(args)
495 - if log_config is not None:
496 - uvicorn_kwargs["log_config"] = log_config
497 -
541 async with build_async_engine_client(
542 args,
543 client_config=client_config,
544 ) as engine_client:
502 - supported_tasks = await engine_client.get_supported_tasks()
503 - logger.info("Supported tasks: %s", supported_tasks)
504 -
505 - app = build_app(args, supported_tasks)
506 - await init_app_state(engine_client, app.state, args, supported_tasks)
507 -
508 - logger.info(
509 - "Starting vLLM API server %d on %s",
510 - engine_client.vllm_config.parallel_config._api_process_rank,
511 - listen_address,
545 + shutdown_task = await build_and_serve(
546 + engine_client, listen_address, sock, args, **uvicorn_kwargs
547 )
513 - shutdown_task = await serve_http(
514 - app,
515 - sock=sock,
516 - enable_ssl_refresh=args.enable_ssl_refresh,
517 - host=args.host,
518 - port=args.port,
519 - log_level=args.uvicorn_log_level,
520 - # NOTE: When the 'disable_uvicorn_access_log' value is True,
521 - # no access log will be output.
522 - access_log=not args.disable_uvicorn_access_log,
523 - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
524 - ssl_keyfile=args.ssl_keyfile,
525 - ssl_certfile=args.ssl_certfile,
526 - ssl_ca_certs=args.ssl_ca_certs,
527 - ssl_cert_reqs=args.ssl_cert_reqs,
528 - ssl_ciphers=args.ssl_ciphers,
529 - h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
530 - h11_max_header_count=args.h11_max_header_count,
531 - **uvicorn_kwargs,
532 - )
533 -
548 # NB: Await server shutdown only after the backend context is exited
549 try:
550 await shutdown_task
vllm/entrypoints/openai/generate/api_router.py
+2 -2
@@ -113,7 +113,7 @@ async def init_generate_state(
113 enable_log_deltas=args.enable_log_deltas,
114 log_error_stack=args.log_error_stack,
115 )
116 - if "generate" in supported_tasks
116 + if any(task in supported_tasks for task in ("generate", "render"))
117 else None
118 )
119 # Warm up chat template processing to avoid first-request latency
@@ -129,7 +129,7 @@ async def init_generate_state(
129 enable_force_include_usage=args.enable_force_include_usage,
130 log_error_stack=args.log_error_stack,
131 )
132 - if "generate" in supported_tasks
132 + if any(task in supported_tasks for task in ("generate", "render"))
133 else None
134 )
135 state.anthropic_serving_messages = (
vllm/tasks.py
+4 -1
@@ -10,4 +10,7 @@ PoolingTask = Literal[
10 ]
11 POOLING_TASKS: tuple[PoolingTask, ...] = get_args(PoolingTask)
12
13 -SupportedTask = Literal[GenerationTask, PoolingTask]
13 +FrontendTask = Literal["render"]
14 +FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask)
15 +
16 +SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask]
vllm/v1/engine/launch.py new
+201
@@ -0,0 +1,201 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +LaunchEngineClient: A lightweight EngineClient for GPU-less online serving.
5 +
6 +This implements the EngineClient protocol without AsyncLLM or EngineCore,
7 +enabling preprocessing (tokenization, rendering) and postprocessing
8 +(detokenization) without GPU inference.
9 +"""
10 +
11 +from collections.abc import AsyncGenerator, Iterable, Mapping
12 +from typing import Any
13 +
14 +from vllm.config import VllmConfig
15 +from vllm.engine.protocol import EngineClient, StreamingInput
16 +from vllm.inputs import ProcessorInputs, PromptType
17 +from vllm.logger import init_logger
18 +from vllm.lora.request import LoRARequest
19 +from vllm.outputs import PoolingRequestOutput, RequestOutput
20 +from vllm.plugins.io_processors import get_io_processor
21 +from vllm.pooling_params import PoolingParams
22 +from vllm.renderers import renderer_from_config
23 +from vllm.sampling_params import SamplingParams
24 +from vllm.tasks import SupportedTask
25 +from vllm.v1.engine import EngineCoreRequest, PauseMode
26 +from vllm.v1.engine.input_processor import InputProcessor
27 +
28 +logger = init_logger(__name__)
29 +
30 +
31 +class LaunchEngineClient(EngineClient):
32 + """GPU-less EngineClient that only supports preprocessing/postprocessing.
33 +
34 + This is a Null Object at the EngineClient level, bypassing AsyncLLM
35 + entirely. It initializes renderer, io_processor, and input_processor
36 + for tokenization and rendering, but raises NotImplementedError for
37 + any inference-related operations.
38 + """
39 +
40 + def __init__(
41 + self,
42 + vllm_config: VllmConfig,
43 + ) -> None:
44 + self.vllm_config = vllm_config
45 + self.model_config = vllm_config.model_config
46 +
47 + self.renderer = renderer = renderer_from_config(self.vllm_config)
48 + self.io_processor = get_io_processor(
49 + self.vllm_config,
50 + self.renderer,
51 + self.model_config.io_processor_plugin,
52 + )
53 +
54 + # Convert TokPrompt --> EngineCoreRequest.
55 + self.input_processor = InputProcessor(self.vllm_config, renderer)
56 +
57 + @classmethod
58 + def from_vllm_config(
59 + cls,
60 + vllm_config: VllmConfig,
61 + ) -> "LaunchEngineClient":
62 + """Create a LaunchEngineClient from a VllmConfig without GPU."""
63 + return cls(
64 + vllm_config=vllm_config,
65 + )
66 +
67 + # -- Task support --
68 +
69 + async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
70 + return ("render",)
71 +
72 + # -- Inference (not supported) --
73 +
74 + async def generate(
75 + self,
76 + prompt: EngineCoreRequest
77 + | PromptType
78 + | ProcessorInputs
79 + | AsyncGenerator[StreamingInput, None],
80 + sampling_params: SamplingParams,
81 + request_id: str,
82 + *,
83 + prompt_text: str | None = None,
84 + lora_request: LoRARequest | None = None,
85 + tokenization_kwargs: dict[str, Any] | None = None,
86 + trace_headers: Mapping[str, str] | None = None,
87 + priority: int = 0,
88 + data_parallel_rank: int | None = None,
89 + reasoning_ended: bool | None = None,
90 + ) -> AsyncGenerator[RequestOutput, None]:
91 + raise NotImplementedError(
92 + "LaunchEngineClient does not support inference. "
93 + "Use vllm serve for generation requests."
94 + )
95 + # yield is needed to make this an async generator
96 + yield # type: ignore[misc] # pragma: no cover
97 +
98 + # -- Request management (no-op) --
99 +
100 + async def abort(
101 + self, request_id: str | Iterable[str], internal: bool = False
102 + ) -> None:
103 + pass
104 +
105 + # -- Generation control (no-op) --
106 +
107 + async def pause_generation(
108 + self,
109 + *,
110 + mode: PauseMode = "abort",
111 + wait_for_inflight_requests: bool | None = None,
112 + clear_cache: bool = True,
113 + ) -> None:
114 + pass
115 +
116 + async def resume_generation(self) -> None:
117 + pass
118 +
119 + async def is_paused(self) -> bool:
120 + return False
121 +
122 + async def encode(
123 + self,
124 + prompt: PromptType | ProcessorInputs,
125 + pooling_params: PoolingParams,
126 + request_id: str,
127 + lora_request: LoRARequest | None = None,
128 + trace_headers: Mapping[str, str] | None = None,
129 + priority: int = 0,
130 + tokenization_kwargs: dict[str, Any] | None = None,
131 + reasoning_ended: bool | None = None,
132 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
133 + raise NotImplementedError(
134 + "LaunchEngineClient does not support inference. "
135 + "Use vllm serve for encoding requests."
136 + )
137 + yield # type: ignore[misc] # pragma: no cover
138 +
139 + # -- Observability (no-op / defaults) --
140 +
141 + async def is_tracing_enabled(self) -> bool:
142 + return False
143 +
144 + async def do_log_stats(self) -> None:
145 + pass
146 +
147 + async def check_health(self) -> None:
148 + pass
149 +
150 + async def start_profile(self) -> None:
151 + pass
152 +
153 + async def stop_profile(self) -> None:
154 + pass
155 +
156 + # -- Cache management (no-op) --
157 +
158 + async def reset_mm_cache(self) -> None:
159 + pass
160 +
161 + async def reset_prefix_cache(
162 + self, reset_running_requests: bool = False, reset_connector: bool = False
163 + ) -> bool:
164 + return True
165 +
166 + async def reset_encoder_cache(self) -> None:
167 + pass
168 +
169 + # -- Power management (no-op) --
170 +
171 + async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
172 + pass
173 +
174 + async def wake_up(self, tags: list[str] | None = None) -> None:
175 + pass
176 +
177 + async def is_sleeping(self) -> bool:
178 + return False
179 +
180 + # -- LoRA (not supported) --
181 +
182 + async def add_lora(self, lora_request: LoRARequest) -> bool:
183 + return False
184 +
185 + # -- Status properties --
186 +
187 + @property
188 + def is_running(self) -> bool:
189 + return True
190 +
191 + @property
192 + def is_stopped(self) -> bool:
193 + return False
194 +
195 + @property
196 + def errored(self) -> bool:
197 + return False
198 +
199 + @property
200 + def dead_error(self) -> BaseException:
201 + return RuntimeError("LaunchEngineClient does not support inference")