@tduncan / Nvidia-Optimization-Stack-VLLM / commits / eac636a7fa

[Frontend] Move api_server.py out openai folder (#52131)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

wang.yuqi committed Aug 19, 2026 at 17:43 UTC eac636a7fa476983cdae34b45a984e9852aad375

43 files changed +887 -775

.buildkite/test-amd.yaml

+6 -4

@@ -990,10 +990,12 @@ steps:

990 - vllm/entrypoints

991 - tests/entrypoints/unit_tests

992 - tests/entrypoints/weight_transfer

993 + - tests/entrypoints/launchers

994 - vllm/platforms/rocm.py

995 commands:

996 - pytest -v -s entrypoints/unit_tests

997 - pytest -v -s entrypoints/weight_transfer

998 + - pytest -v -s entrypoints/launchers

999

1000 - label: Entrypoints Integration (LLM) # TBD

1001 timeout_in_minutes: 180

@@ -2320,9 +2322,9 @@ steps:

2322 - tests/benchmarks/test_serve_cli.py

2323 - tests/entrypoints/openai/chat_completion/test_chat_completion.py

2324 - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py

2323 - - tests/entrypoints/openai/completion/test_shutdown.py

2325 + - tests/entrypoints/launchers/test_shutdown.py

2326 - tests/entrypoints/openai/test_return_token_ids.py

2325 - - tests/entrypoints/openai/test_uds.py

2327 + - tests/entrypoints/serve/instrumentator/test_uds.py

2328 - tests/v1/sample/test_logprobs_e2e.py

2329 - vllm/platforms/rocm.py

2330 commands:

@@ -2331,9 +2333,9 @@ steps:

2333 - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"

2334 - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"

2335 - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"

2334 - - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"

2336 + - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"

2337 - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"

2336 - - pytest -v -s entrypoints/openai/test_uds.py

2338 + - pytest -v -s entrypoints/serve/instrumentator/test_uds.py

2339 - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"

2340

2341 - label: Rust Frontend Serve Admin Coverage # TBD

.buildkite/test_areas/entrypoints.yaml

@@ -11,9 +11,11 @@ steps:

11 - vllm/entrypoints

12 - tests/entrypoints/unit_tests

13 - tests/entrypoints/weight_transfer

14 + - tests/entrypoints/launchers

15 commands:

16 - pytest -v -s entrypoints/unit_tests

17 - pytest -v -s entrypoints/weight_transfer

18 + - pytest -v -s entrypoints/launchers

19 mirror:

20 amd:

21 label: ":amd: (MI355) Entrypoints Unit"

.buildkite/test_areas/rust_frontend.yaml

+4 -4

@@ -18,9 +18,9 @@ steps:

18 - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py

20 # - tests/entrypoints/openai/completion/test_prompt_validation.py

21 - - tests/entrypoints/openai/completion/test_shutdown.py

21 + - tests/entrypoints/launchers/test_shutdown.py

22 - tests/entrypoints/openai/test_return_token_ids.py

23 - - tests/entrypoints/openai/test_uds.py

23 + - tests/entrypoints/serve/instrumentator/test_uds.py

24 - tests/v1/sample/test_logprobs_e2e.py

25 commands:

26 - export VLLM_USE_RUST_FRONTEND=1

@@ -30,11 +30,11 @@ steps:

30 - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"

32 # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"

33 - - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"

33 + - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"

34 # test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and

35 # finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk.

36 - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"

37 - - pytest -v -s entrypoints/openai/test_uds.py

37 + - pytest -v -s entrypoints/serve/instrumentator/test_uds.py

38 - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"

40 - label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"

docs/deployment/integrations/kthena.md

+1 -1

@@ -117,7 +117,7 @@ Commands:

117 - >

118 bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh

119 leader --ray_cluster_size=2; python3 -m

120 - vllm.entrypoints.openai.api_server --port 8080 --model

120 + vllm.entrypoints.launchers.api_server.entry --port 8080 --model

121 meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8

122 --pipeline-parallel-size 2

123 ```

docs/design/arch_overview.md

+1 -15

@@ -62,20 +62,6 @@ vllm serve <model>

63 The code for the `vllm` CLI can be found in [vllm/entrypoints/cli/main.py](../../vllm/entrypoints/cli/main.py).

65 -Sometimes you may see the API server entrypoint used directly instead of via the

66 -`vllm` CLI command. For example:

67 -

68 -```bash

69 -python -m vllm.entrypoints.openai.api_server --model <model>

70 -```

71 -

72 -!!! warning

73 -

74 - `python -m vllm.entrypoints.openai.api_server` is deprecated

75 - and may become unsupported in a future release.

76 -

77 -That code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py).

78 -

65 More details on the API server can be found in the [Online Serving](../serving/online_serving/README.md) document.

67 ## V1 Process Architecture

@@ -88,7 +74,7 @@ The API server process handles HTTP requests (e.g., the OpenAI-compatible API),

75 By default, there is **1 API server process**, but when data parallelism is used, the API server count automatically scales to match the data parallel size. This can also be manually configured with the `--api-server-count` flag. Each API server connects to **all** engine cores via ZMQ in a many-to-many topology, enabling any API server to route requests to any engine core. Each API server process uses multiple CPU threads for media loading (controlled by `VLLM_MEDIA_LOADING_THREAD_COUNT`, default 8).

91 -The code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py) and [vllm/v1/utils.py](../../vllm/v1/utils.py).

77 +The code can be found in [vllm/entrypoints/launchers/api_server](../../vllm/entrypoints/launchers/api_server) and [vllm/v1/utils.py](../../vllm/v1/utils.py).

79 ### Engine Core Process

docs/design/endpoint_plugins.md

+1 -1

@@ -1,6 +1,6 @@

1 # Endpoint Plugins

3 -Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server without editing `vllm/entrypoints/openai/api_server.py`. Their scope is

3 +Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server. Their scope is

4 the **HTTP surface only** registering routes and optionally per app state used by those routes. A plugin reaches the engine the same way an in-tree serving handler does, through the `EngineClient` it is handed at startup (e.g. `engine_client.collective_rpc(...)`). No new engine access path is introduced.

6 !!! warning "Security"

examples/applications/api_server/server.py

+2 -3

@@ -4,8 +4,7 @@

4 NOTE: This API server is used only for demonstrating usage of AsyncEngine

5 and simple performance benchmarks. It is not intended for production use.

6 For production use, we recommend using our OpenAI compatible server.

7 -We are also not going to accept PRs modifying this file, please

8 -change `vllm/entrypoints/openai/api_server.py` instead.

7 +We are also not going to accept PRs modifying this file.

8 """

10 import asyncio

@@ -21,7 +20,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse

20 import vllm.envs as envs

21 from vllm.engine.arg_utils import AsyncEngineArgs

22 from vllm.engine.async_llm_engine import AsyncLLMEngine

24 -from vllm.entrypoints.launcher import serve_http

23 +from vllm.entrypoints.launchers.launcher import serve_http

24 from vllm.entrypoints.serve.utils.api_utils import with_cancellation

25 from vllm.logger import init_logger

26 from vllm.sampling_params import SamplingParams

tests/entrypoints/launchers/__init__.py

tests/entrypoints/launchers/api_server/__init__.py

tests/entrypoints/launchers/api_server/_api_server_spawn_workers.py renamed

tests/entrypoints/launchers/api_server/test_api_server_process_manager.py renamed

+1 -1

@@ -10,7 +10,7 @@ from unittest.mock import patch

10 import pytest

11 import zmq

13 -from tests.entrypoints.unit_tests._api_server_spawn_workers import (

13 +from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (

14 exit_before_report_worker,

15 )

16 from vllm.utils.network_utils import make_zmq_socket, split_zmq_path

tests/entrypoints/launchers/api_server/test_multi_api_servers.py renamed

tests/entrypoints/launchers/test_launch_cli.py renamed

tests/entrypoints/launchers/test_shutdown.py renamed

+1 -1

@@ -147,7 +147,7 @@ async def test_shutdown_on_engine_failure(tmp_path: Path):

147 # dtype, max-len etc set so that this can run in CI

148 sys.executable,

149 "-m",

150 - "vllm.entrypoints.openai.api_server",

150 + "vllm.entrypoints.launchers.api_server.entry",

151 "--model",

152 MODEL_NAME,

153 "--dtype",

tests/entrypoints/launchers/test_ssl_cert_refresher.py renamed

+1 -1

@@ -7,7 +7,7 @@ from ssl import SSLContext

8 import pytest

10 -from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher

10 +from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher

13 class MockSSLContext(SSLContext):

tests/entrypoints/serve/exception_handling/test_http_status_metrics.py

+1 -1

@@ -14,7 +14,7 @@ import pytest

14 from fastapi import HTTPException

15 from prometheus_client import CollectorRegistry

17 -from vllm.entrypoints.openai.api_server import build_app

17 +from vllm.entrypoints.launchers.api_server.entry import build_app

18 from vllm.exceptions import (

19 VLLMNotFoundError,

20 VLLMServerError,

tests/entrypoints/serve/instrumentator/test_uds.py renamed

+1 -2

@@ -6,11 +6,10 @@ from tempfile import TemporaryDirectory

6 import httpx

7 import pytest

9 +from tests.utils import RemoteOpenAIServer

10 from vllm import envs

11 from vllm.version import __version__ as VLLM_VERSION

12 -from ...utils import RemoteOpenAIServer

13 -

13 MODEL_NAME = "Qwen/Qwen3-0.6B"

tests/lora/test_add_lora.py

+1 -1

@@ -6,7 +6,7 @@ import time

6 import pytest

8 from vllm.engine.arg_utils import AsyncEngineArgs

9 -from vllm.entrypoints.openai.api_server import (

9 +from vllm.entrypoints.launchers.api_server.entry import (

10 build_async_engine_client_from_engine_args,

11 )

12 from vllm.inputs import TextPrompt

tests/lora/test_lora_functions.py

+1 -1

@@ -7,7 +7,7 @@ Script to test add_lora, remove_lora, pin_lora, list_loras functions.

7 import pytest

9 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs

10 -from vllm.entrypoints.openai.api_server import (

10 +from vllm.entrypoints.launchers.api_server.entry import (

11 build_async_engine_client_from_engine_args,

12 )

13 from vllm.lora.request import LoRARequest

tests/plugins_tests/test_endpoint_plugins.py

+10 -10

@@ -17,14 +17,14 @@ import pytest

17 from fastapi import FastAPI

18 from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin

20 -from vllm.entrypoints.openai.api_server import (

21 - _attach_endpoint_plugins,

22 - _init_endpoint_plugins_state,

23 - build_app,

24 -)

20 +from vllm.entrypoints.launchers.app import build_app

21 from vllm.entrypoints.openai.cli_args import make_arg_parser

22 from vllm.plugins import load_endpoint_plugins

27 -from vllm.plugins.endpoint_plugins.interface import EndpointPlugin

23 +from vllm.plugins.endpoint_plugins.interface import (

24 + EndpointPlugin,

25 + attach_endpoint_plugins,

26 + init_endpoint_plugins_state,

27 +)

28 from vllm.utils.argparse_utils import FlexibleArgumentParser

@@ -157,7 +157,7 @@ def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch)

157 monkeypatch.delenv("VLLM_PLUGINS", raising=False)

158

159 app = FastAPI()

160 - _attach_endpoint_plugins(app, ("generate",))

160 + attach_endpoint_plugins(app, ("generate",))

161

162 assert app.state.endpoint_plugins == []

163

@@ -172,7 +172,7 @@ async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatc

172 monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")

173

174 state = State()

175 - await _init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())

175 + await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())

176

177 assert not hasattr(state, "dummy_engine_client")

178

@@ -196,7 +196,7 @@ async def test_render_server_attaches_endpoint_plugins_with_no_engine_client(

196 for route in app.routes

197 )

198

199 - await _init_endpoint_plugins_state(None, app.state, args)

199 + await init_endpoint_plugins_state(None, app.state, args)

200

201 assert app.state.dummy_engine_client is None

202

@@ -223,7 +223,7 @@ async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch):

223 )

224

225 fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])

226 - await _init_endpoint_plugins_state(fake_engine_client, app.state, args)

226 + await init_endpoint_plugins_state(fake_engine_client, app.state, args)

227

228 assert app.state.dummy_engine_client is fake_engine_client

229

vllm/benchmarks/throughput.py

+1 -1

@@ -305,7 +305,7 @@ async def run_vllm_async(

305 disable_detokenize: bool = False,

306 warmup_requests: list[SampleRequest] | None = None,

307 ) -> float:

308 - from vllm.entrypoints.openai.api_server import (

308 + from vllm.entrypoints.launchers.api_server.entry import (

309 build_async_engine_client_from_engine_args,

310 )

311

vllm/entrypoints/cli/launch.py

+1 -43

@@ -3,18 +3,11 @@

4 import argparse

5 import inspect

6 -import signal

7 import uvloop

10 -from vllm import envs

11 -from vllm.config import VllmConfig

12 -from vllm.engine.arg_utils import AsyncEngineArgs

9 from vllm.entrypoints.cli.types import CLISubcommand

14 -from vllm.entrypoints.openai.api_server import (

15 - build_and_serve_renderer,

16 - setup_server,

17 -)

10 +from vllm.entrypoints.launchers.render.entry import run_launch_fastapi

11 from vllm.entrypoints.openai.cli_args import (

12 make_arg_parser,

13 validate_parsed_serve_args,

@@ -120,38 +113,3 @@ class LaunchSubcommand(CLISubcommand):

113

114 def cmd_init() -> list[CLISubcommand]:

115 return [LaunchSubcommand()]

123 -

124 -

125 -async def run_launch_fastapi(args: argparse.Namespace) -> None:

126 - """Run the online serving layer with FastAPI (no GPU inference)."""

127 -

128 - # Interrupt initialization if SIGTERM arrives before uvicorn installs

129 - # its own signal handlers. Once uvicorn is running it replaces this.

130 - def _interrupt_init(*_) -> None:

131 - raise KeyboardInterrupt("terminated")

132 -

133 - signal.signal(signal.SIGTERM, _interrupt_init)

134 -

135 - # 1. Socket binding

136 - listen_address, sock = setup_server(args, reuse_port=False)

137 -

138 - # 2. Build and serve the API server

139 - engine_args = AsyncEngineArgs.from_cli_args(args)

140 - model_config = engine_args.create_model_config()

141 -

142 - # Render servers preprocess data only — no inference, no quantized kernels.

143 - # Clear quantization so VllmConfig skips quant dtype/capability validation.

144 - model_config.quantization = None

145 -

146 - # Render servers never allocate KV cache; suppress the spurious CPU KV

147 - # cache space warning from CpuPlatform.check_and_update_config.

148 - envs.VLLM_CPU_KVCACHE_SPACE = 0

149 -

150 - vllm_config = VllmConfig(model_config=model_config)

151 - shutdown_task = await build_and_serve_renderer(

152 - vllm_config, listen_address, sock, args

153 - )

154 - try:

155 - await shutdown_task

156 - finally:

157 - sock.close()

vllm/entrypoints/cli/serve.py

+2 -4

@@ -10,11 +10,9 @@ import uvloop

10 import vllm

11 import vllm.envs as envs

12 from vllm.entrypoints.cli.types import CLISubcommand

13 -from vllm.entrypoints.openai.api_server import run_server, setup_server

13 +from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server

14 from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args

15 -from vllm.entrypoints.openai.dp_supervisor import (

16 - run_dp_supervisor,

17 -)

15 +from vllm.entrypoints.openai.dp_supervisor import run_dp_supervisor

16 from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG

17 from vllm.logger import init_logger

18 from vllm.usage.usage_lib import UsageContext

vllm/entrypoints/launchers/api_server/app_state.py new

+149

@@ -0,0 +1,149 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import warnings

4 +from argparse import Namespace

5 +from typing import cast

6 +

7 +from starlette.datastructures import State

8 +

9 +from vllm.engine.protocol import EngineClient

10 +from vllm.entrypoints.chat_utils import load_chat_template

11 +from vllm.entrypoints.openai.models.protocol import BaseModelPath

12 +from vllm.entrypoints.openai.models.serving import OpenAIServingModels

13 +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization

14 +from vllm.entrypoints.serve.utils.api_utils import process_lora_modules

15 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger

16 +from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state

17 +from vllm.renderers.online_derenderer import OnlineDerenderer

18 +from vllm.renderers.online_renderer import OnlineRenderer

19 +from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask

20 +

21 +

22 +async def init_app_state(

23 + engine_client: EngineClient,

24 + state: State,

25 + args: Namespace,

26 + supported_tasks: tuple["SupportedTask", ...] | None = None,

27 +) -> None:

28 + vllm_config = engine_client.vllm_config

29 +

30 + if args.tool_call_parser is not None:

31 + from vllm.parser.metrics import init_parser_metrics

32 +

33 + init_parser_metrics(

34 + model_name=cast(str, vllm_config.model_config.served_model_name)

35 + )

36 +

37 + if supported_tasks is None:

38 + warnings.warn(

39 + "The 'supported_tasks' parameter was not provided to "

40 + "init_app_state and will be required in a future version. "

41 + "Please pass 'supported_tasks' explicitly.",

42 + DeprecationWarning,

43 + stacklevel=2,

44 + )

45 + supported_tasks = FALLBACK_SUPPORTED_TASKS

46 +

47 + if args.served_model_name is not None:

48 + served_model_names = args.served_model_name

49 + else:

50 + served_model_names = [args.model]

51 +

52 + if args.enable_log_requests:

53 + request_logger = RequestLogger(max_log_len=args.max_log_len)

54 + else:

55 + request_logger = None

56 +

57 + base_model_paths = [

58 + BaseModelPath(name=name, model_path=args.model) for name in served_model_names

59 + ]

60 +

61 + state.engine_client = engine_client

62 + state.log_stats = not args.disable_log_stats

63 + state.vllm_config = vllm_config

64 + state.args = args

65 + resolved_chat_template = load_chat_template(args.chat_template)

66 +

67 + # Merge default_mm_loras into the static lora_modules

68 + default_mm_loras = (

69 + vllm_config.lora_config.default_mm_loras

70 + if vllm_config.lora_config is not None

71 + else {}

72 + )

73 + lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)

74 +

75 + state.openai_serving_models = OpenAIServingModels(

76 + engine_client=engine_client,

77 + base_model_paths=base_model_paths,

78 + lora_modules=lora_modules,

79 + )

80 + await state.openai_serving_models.init_static_loras()

81 +

82 + state.online_renderer = OnlineRenderer(

83 + model_config=engine_client.model_config,

84 + renderer=engine_client.renderer,

85 + request_logger=request_logger,

86 + chat_template=resolved_chat_template,

87 + chat_template_content_format=args.chat_template_content_format,

88 + trust_request_chat_template=args.trust_request_chat_template,

89 + enable_auto_tools=args.enable_auto_tool_choice,

90 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

91 + tool_parser=args.tool_call_parser,

92 + reasoning_parser=args.structured_outputs_config.reasoning_parser,

93 + default_chat_template_kwargs=args.default_chat_template_kwargs,

94 + log_error_stack=args.log_error_stack,

95 + )

96 + state.online_renderer.warmup()

97 +

98 + state.online_derenderer = OnlineDerenderer(

99 + model_config=engine_client.model_config,

100 + renderer=engine_client.renderer,

101 + request_logger=request_logger,

102 + chat_template=resolved_chat_template,

103 + chat_template_content_format=args.chat_template_content_format,

104 + trust_request_chat_template=args.trust_request_chat_template,

105 + enable_auto_tools=args.enable_auto_tool_choice,

106 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

107 + tool_parser=args.tool_call_parser,

108 + reasoning_parser=args.structured_outputs_config.reasoning_parser,

109 + default_chat_template_kwargs=args.default_chat_template_kwargs,

110 + log_error_stack=args.log_error_stack,

111 + )

112 +

113 + state.serving_tokenization = ServingTokenization(

114 + state.openai_serving_models,

115 + state.online_renderer,

116 + request_logger=request_logger,

117 + chat_template=resolved_chat_template,

118 + chat_template_content_format=args.chat_template_content_format,

119 + default_chat_template_kwargs=args.default_chat_template_kwargs,

120 + trust_request_chat_template=args.trust_request_chat_template,

121 + )

122 +

123 + if "generate" in supported_tasks:

124 + from vllm.entrypoints.generate.api_router import init_generate_state

125 +

126 + await init_generate_state(

127 + engine_client, state, args, request_logger, supported_tasks

128 + )

129 +

130 + from vllm.entrypoints.scale_out.factories import init_scale_out_state

131 +

132 + init_scale_out_state(state, args, engine_client, request_logger)

133 +

134 + if "transcription" in supported_tasks or "realtime" in supported_tasks:

135 + from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state

136 +

137 + init_speech_to_text_state(

138 + engine_client, state, args, request_logger, supported_tasks

139 + )

140 +

141 + if any(task in POOLING_TASKS for task in supported_tasks):

142 + from vllm.entrypoints.pooling.factories import init_pooling_state

143 +

144 + init_pooling_state(engine_client, state, args, request_logger, supported_tasks)

145 +

146 + await init_endpoint_plugins_state(engine_client, state, args)

147 +

148 + state.enable_server_load_tracking = args.enable_server_load_tracking

149 + state.server_load_metrics = 0

vllm/entrypoints/launchers/api_server/entry.py new

+229

@@ -0,0 +1,229 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import asyncio

4 +import multiprocessing

5 +import multiprocessing.forkserver as forkserver

6 +import os

7 +import signal

8 +import socket

9 +import tempfile

10 +from argparse import Namespace

11 +from collections.abc import AsyncIterator

12 +from contextlib import asynccontextmanager

13 +from typing import Any

14 +

15 +import vllm.envs as envs

16 +from vllm.engine.arg_utils import AsyncEngineArgs

17 +from vllm.engine.protocol import EngineClient

18 +from vllm.logger import init_logger

19 +from vllm.reasoning import ReasoningParserManager

20 +from vllm.tool_parsers import ToolParserManager

21 +from vllm.usage.usage_lib import UsageContext

22 +from vllm.utils.system_utils import decorate_logs

23 +

24 +from ..app import build_app

25 +from ..launcher import serve_http, setup_server

26 +from ..utils.server_utils import get_uvicorn_log_config

27 +from .app_state import init_app_state

28 +

29 +prometheus_multiproc_dir: tempfile.TemporaryDirectory

30 +

31 +# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)

32 +logger = init_logger("vllm.entrypoints.launchers.api_server.entry")

33 +

34 +

35 +@asynccontextmanager

36 +async def build_async_engine_client(

37 + args: Namespace,

38 + *,

39 + usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,

40 + client_config: dict[str, Any] | None = None,

41 +) -> AsyncIterator[EngineClient]:

42 + if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":

43 + # The executor is expected to be mp.

44 + # Pre-import heavy modules in the forkserver process

45 + logger.debug("Setup forkserver with pre-imports")

46 + multiprocessing.set_start_method("forkserver")

47 + multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])

48 + forkserver.ensure_running()

49 + logger.debug("Forkserver setup complete!")

50 +

51 + # Context manager to handle engine_client lifecycle

52 + # Ensures everything is shutdown and cleaned up on error/exit

53 + engine_args = AsyncEngineArgs.from_cli_args(args)

54 + if client_config:

55 + engine_args._api_process_count = client_config.get("client_count", 1)

56 + engine_args._api_process_rank = client_config.get("client_index", 0)

57 +

58 + async with build_async_engine_client_from_engine_args(

59 + engine_args,

60 + usage_context=usage_context,

61 + client_config=client_config,

62 + ) as engine:

63 + yield engine

64 +

65 +

66 +@asynccontextmanager

67 +async def build_async_engine_client_from_engine_args(

68 + engine_args: AsyncEngineArgs,

69 + *,

70 + usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,

71 + client_config: dict[str, Any] | None = None,

72 +) -> AsyncIterator[EngineClient]:

73 + """

74 + Create EngineClient, either:

75 + - in-process using the AsyncLLMEngine Directly

76 + - multiprocess using AsyncLLMEngine RPC

77 +

78 + Returns the Client or None if the creation failed.

79 + """

80 +

81 + # Create the EngineConfig (determines if we can use V1).

82 + vllm_config = engine_args.create_engine_config(usage_context=usage_context)

83 +

84 + from vllm.v1.engine.async_llm import AsyncLLM

85 +

86 + async_llm: AsyncLLM | None = None

87 +

88 + # Don't mutate the input client_config

89 + client_config = dict(client_config) if client_config else {}

90 + client_count = client_config.pop("client_count", 1)

91 + client_index = client_config.pop("client_index", 0)

92 +

93 + try:

94 + async_llm = AsyncLLM.from_vllm_config(

95 + vllm_config=vllm_config,

96 + usage_context=usage_context,

97 + enable_log_requests=engine_args.enable_log_requests,

98 + aggregate_engine_logging=engine_args.aggregate_engine_logging,

99 + disable_log_stats=engine_args.disable_log_stats,

100 + client_addresses=client_config,

101 + client_count=client_count,

102 + client_index=client_index,

103 + )

104 +

105 + # Don't keep the dummy data in memory

106 + assert async_llm is not None

107 + await async_llm.reset_mm_cache()

108 +

109 + yield async_llm

110 + finally:

111 + if async_llm:

112 + async_llm.shutdown(timeout=vllm_config.shutdown_timeout)

113 +

114 +

115 +async def build_and_serve(

116 + engine_client: EngineClient,

117 + listen_address: str,

118 + sock: socket.socket,

119 + args: Namespace,

120 + **uvicorn_kwargs,

121 +) -> asyncio.Task:

122 + """Build FastAPI app, initialize state, and start serving.

123 +

124 + Returns the shutdown task for the caller to await.

125 + """

126 +

127 + # Get uvicorn log config (from file or with endpoint filter)

128 + log_config = get_uvicorn_log_config(args)

129 + if log_config is not None:

130 + uvicorn_kwargs["log_config"] = log_config

131 +

132 + supported_tasks = await engine_client.get_supported_tasks()

133 + model_config = engine_client.model_config

134 +

135 + logger.info("Supported tasks: %s", supported_tasks)

136 + app = build_app(args, supported_tasks, model_config)

137 + await init_app_state(engine_client, app.state, args, supported_tasks)

138 +

139 + logger.info("Starting vLLM server on %s", listen_address)

140 +

141 + return await serve_http(

142 + app,

143 + sock=sock,

144 + enable_ssl_refresh=args.enable_ssl_refresh,

145 + host=args.host,

146 + port=args.port,

147 + log_level=args.uvicorn_log_level,

148 + # NOTE: When the 'disable_uvicorn_access_log' value is True,

149 + # no access log will be output.

150 + access_log=not args.disable_uvicorn_access_log,

151 + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,

152 + ssl_keyfile=args.ssl_keyfile,

153 + ssl_certfile=args.ssl_certfile,

154 + ssl_ca_certs=args.ssl_ca_certs,

155 + ssl_cert_reqs=args.ssl_cert_reqs,

156 + ssl_ciphers=args.ssl_ciphers,

157 + h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,

158 + h11_max_header_count=args.h11_max_header_count,

159 + **uvicorn_kwargs,

160 + )

161 +

162 +

163 +async def run_server(args, **uvicorn_kwargs) -> None:

164 + """Run a single-worker API server."""

165 +

166 + decorate_logs("APIServer", skip_if_decorated=True)

167 +

168 + # Interrupt initialization if SIGTERM arrives before uvicorn installs its

169 + # own signal handlers. Once uvicorn is running it replaces this.

170 + def _interrupt_init(*_) -> None:

171 + raise KeyboardInterrupt("terminated")

172 +

173 + signal.signal(signal.SIGTERM, _interrupt_init)

174 +

175 + listen_address, sock = setup_server(args, reuse_port=False)

176 + await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)

177 +

178 +

179 +async def run_server_worker(

180 + listen_address, sock, args, client_config=None, **uvicorn_kwargs

181 +) -> None:

182 + """Run a single API server worker."""

183 +

184 + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:

185 + ToolParserManager.import_tool_parser(args.tool_parser_plugin)

186 +

187 + if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:

188 + ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

189 +

190 + async with build_async_engine_client(

191 + args,

192 + client_config=client_config,

193 + ) as engine_client:

194 + shutdown_task = await build_and_serve(

195 + engine_client, listen_address, sock, args, **uvicorn_kwargs

196 + )

197 + # NB: Await server shutdown only after the backend context is exited

198 + try:

199 + await shutdown_task

200 + finally:

201 + sock.close()

202 +

203 +

204 +def main():

205 + import uvloop

206 +

207 + from vllm.entrypoints.openai.cli_args import (

208 + make_arg_parser,

209 + validate_parsed_serve_args,

210 + )

211 + from vllm.entrypoints.serve.utils.api_utils import cli_env_setup

212 + from vllm.utils.argparse_utils import FlexibleArgumentParser

213 +

214 + # NOTE(simon):

215 + # This section should be in sync with vllm/entrypoints/cli/main.py for CLI

216 + # entrypoints.

217 + cli_env_setup()

218 + parser = FlexibleArgumentParser(

219 + description="vLLM OpenAI-Compatible RESTful API server."

220 + )

221 + parser = make_arg_parser(parser)

222 + args = parser.parse_args()

223 + validate_parsed_serve_args(args)

224 +

225 + uvloop.run(run_server(args))

226 +

227 +

228 +if __name__ == "__main__":

229 + main()

vllm/entrypoints/launchers/app.py new

+56

@@ -0,0 +1,56 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import warnings

4 +from argparse import Namespace

5 +

6 +from fastapi import FastAPI

7 +

8 +from vllm.config import ModelConfig

9 +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler

10 +from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware

11 +from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap

12 +from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins

13 +from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask

14 +

15 +from .api_server.routers import register_api_routers

16 +from .utils.server_utils import lifespan

17 +

18 +

19 +def build_app(

20 + args: Namespace,

21 + supported_tasks: tuple["SupportedTask", ...] | None = None,

22 + model_config: ModelConfig | None = None,

23 +) -> FastAPI:

24 + if supported_tasks is None:

25 + warnings.warn(

26 + "The 'supported_tasks' parameter was not provided to "

27 + "build_app and will be required in a future version. "

28 + "Defaulting to ('generate',).",

29 + DeprecationWarning,

30 + stacklevel=2,

31 + )

32 + supported_tasks = FALLBACK_SUPPORTED_TASKS

33 +

34 + if args.disable_fastapi_docs:

35 + app = FastAPI(

36 + openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan

37 + )

38 + elif args.enable_offline_docs:

39 + app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)

40 + else:

41 + app = FastAPI(lifespan=lifespan)

42 + app.state.args = args

43 + app.root_path = args.root_path

44 +

45 + register_api_routers(args, app, supported_tasks, model_config)

46 +

47 + # Endpoint plugins are attached last so their routes are registered after all core

48 + # routers. This runs even for the CPU only render server. A plugin eligible for

49 + # the `render` task still gets its routes registered. It receives

50 + # `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).

51 + attach_endpoint_plugins(app, supported_tasks)

52 +

53 + init_exception_handler(app)

54 + init_entrypoints_middleware(args, app, supported_tasks)

55 + app = sagemaker_standards_bootstrap(app)

56 + return app

vllm/entrypoints/launchers/launcher.py renamed

+94 -4

@@ -14,13 +14,23 @@ from fastapi import FastAPI

15 from vllm import envs

16 from vllm.engine.protocol import EngineClient

17 -from vllm.entrypoints.serve.utils.constants import (

17 +from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher

18 +from vllm.entrypoints.serve.utils.api_utils import (

19 + log_non_default_args,

20 + log_version_and_model,

21 +)

22 +from vllm.logger import init_logger

23 +from vllm.reasoning import ReasoningParserManager

24 +from vllm.tool_parsers import ToolParserManager

25 +from vllm.tracing import instrument

26 +from vllm.utils.network_utils import find_process_using_port, is_valid_ipv6_address

27 +from vllm.utils.system_utils import set_ulimit

28 +from vllm.version import __version__ as VLLM_VERSION

29 +

30 +from .utils.constants import (

31 H11_MAX_HEADER_COUNT_DEFAULT,

32 H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,

33 )

21 -from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher

22 -from vllm.logger import init_logger

23 -from vllm.utils.network_utils import find_process_using_port

35 logger = init_logger(__name__)

@@ -190,3 +200,83 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):

200 engine_errored = engine.errored and not engine.is_running

201 if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:

202 server.should_exit = True

203 +

204 +

205 +def create_server_socket(

206 + addr: tuple[str, int],

207 + *,

208 + reuse_port: bool,

209 +) -> socket.socket:

210 + family = socket.AF_INET

211 + if is_valid_ipv6_address(addr[0]):

212 + family = socket.AF_INET6

213 +

214 + sock = socket.socket(family=family, type=socket.SOCK_STREAM)

215 + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

216 + if reuse_port:

217 + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)

218 + sock.bind(addr)

219 +

220 + return sock

221 +

222 +

223 +def create_server_unix_socket(path: str) -> socket.socket:

224 + sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)

225 + sock.bind(path)

226 + return sock

227 +

228 +

229 +def validate_api_server_args(args):

230 + valid_tool_parses = ToolParserManager.list_registered()

231 + if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:

232 + raise KeyError(

233 + f"invalid tool call parser: {args.tool_call_parser} "

234 + f"(chose from {{ {','.join(valid_tool_parses)} }})"

235 + )

236 +

237 + valid_reasoning_parsers = ReasoningParserManager.list_registered()

238 + if (

239 + reasoning_parser := args.structured_outputs_config.reasoning_parser

240 + ) and reasoning_parser not in valid_reasoning_parsers:

241 + raise KeyError(

242 + f"invalid reasoning parser: {reasoning_parser} "

243 + f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"

244 + )

245 +

246 +

247 +@instrument(span_name="API server setup")

248 +def setup_server(args, *, reuse_port: bool):

249 + """Validate API server args and create the server socket."""

250 +

251 + log_version_and_model(logger, VLLM_VERSION, args.model)

252 + log_non_default_args(args)

253 +

254 + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:

255 + ToolParserManager.import_tool_parser(args.tool_parser_plugin)

256 +

257 + if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:

258 + ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

259 +

260 + validate_api_server_args(args)

261 +

262 + # workaround to make sure that we bind the port before the engine is set up.

263 + # This avoids race conditions with ray.

264 + # see https://github.com/vllm-project/vllm/issues/8204

265 + if args.uds:

266 + sock = create_server_unix_socket(args.uds)

267 + else:

268 + sock_addr = (args.host or "", args.port)

269 + sock = create_server_socket(sock_addr, reuse_port=reuse_port)

270 +

271 + # workaround to avoid footguns where uvicorn drops requests with too

272 + # many concurrent requests active

273 + set_ulimit()

274 +

275 + if args.uds:

276 + listen_address = f"unix:{args.uds}"

277 + else:

278 + addr, port = sock_addr

279 + is_ssl = args.ssl_keyfile and args.ssl_certfile

280 + host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"

281 + listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"

282 + return listen_address, sock

vllm/entrypoints/launchers/render/__init__.py

vllm/entrypoints/launchers/render/app_state.py new

+104

@@ -0,0 +1,104 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from argparse import Namespace

4 +

5 +from starlette.datastructures import State

6 +

7 +from vllm.config import VllmConfig

8 +from vllm.entrypoints.chat_utils import load_chat_template

9 +from vllm.entrypoints.openai.models.protocol import BaseModelPath

10 +from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry

11 +from vllm.entrypoints.scale_out.factories import init_render_state

12 +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization

13 +from vllm.entrypoints.serve.utils.request_logger import RequestLogger

14 +from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state

15 +from vllm.renderers import renderer_from_config

16 +from vllm.renderers.online_derenderer import OnlineDerenderer

17 +from vllm.renderers.online_renderer import OnlineRenderer

18 +

19 +

20 +async def init_render_app_state(

21 + vllm_config: VllmConfig,

22 + state: State,

23 + args: Namespace,

24 +) -> None:

25 + """Initialise FastAPI app state for a CPU-only render server.

26 +

27 + Unlike :func:`init_app_state` this function does not require an

28 + :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the

29 + preprocessing pipeline (renderer, input_processor)

30 + directly from the :class:`~vllm.config.VllmConfig`.

31 + """

32 +

33 + served_model_names = args.served_model_name or [args.model]

34 + model_registry = OpenAIModelRegistry(

35 + model_config=vllm_config.model_config,

36 + base_model_paths=[

37 + BaseModelPath(name=name, model_path=args.model)

38 + for name in served_model_names

39 + ],

40 + )

41 +

42 + if args.enable_log_requests:

43 + request_logger = RequestLogger(max_log_len=args.max_log_len)

44 + else:

45 + request_logger = None

46 +

47 + renderer = renderer_from_config(vllm_config)

48 + resolved_chat_template = load_chat_template(args.chat_template)

49 +

50 + state.online_renderer = OnlineRenderer(

51 + model_config=vllm_config.model_config,

52 + renderer=renderer,

53 + request_logger=request_logger,

54 + chat_template=resolved_chat_template,

55 + chat_template_content_format=args.chat_template_content_format,

56 + trust_request_chat_template=args.trust_request_chat_template,

57 + enable_auto_tools=args.enable_auto_tool_choice,

58 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

59 + tool_parser=args.tool_call_parser,

60 + reasoning_parser=args.reasoning_parser,

61 + default_chat_template_kwargs=args.default_chat_template_kwargs,

62 + log_error_stack=args.log_error_stack,

63 + )

64 + state.online_renderer.warmup()

65 +

66 + state.online_derenderer = OnlineDerenderer(

67 + model_config=vllm_config.model_config,

68 + renderer=renderer,

69 + request_logger=request_logger,

70 + chat_template=resolved_chat_template,

71 + chat_template_content_format=args.chat_template_content_format,

72 + trust_request_chat_template=args.trust_request_chat_template,

73 + enable_auto_tools=args.enable_auto_tool_choice,

74 + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

75 + tool_parser=args.tool_call_parser,

76 + reasoning_parser=args.reasoning_parser,

77 + default_chat_template_kwargs=args.default_chat_template_kwargs,

78 + log_error_stack=args.log_error_stack,

79 + )

80 +

81 + state.openai_serving_models = model_registry

82 + state.serving_tokenization = ServingTokenization(

83 + model_registry,

84 + state.online_renderer,

85 + request_logger=request_logger,

86 + chat_template=resolved_chat_template,

87 + chat_template_content_format=args.chat_template_content_format,

88 + default_chat_template_kwargs=args.default_chat_template_kwargs,

89 + trust_request_chat_template=args.trust_request_chat_template,

90 + )

91 +

92 + init_render_state(state, request_logger)

93 +

94 + state.vllm_config = vllm_config

95 + # Disable stats logging — there is no engine to poll.

96 + state.log_stats = False

97 + state.engine_client = None

98 + state.args = args

99 + state.enable_server_load_tracking = False

100 + state.server_load_metrics = 0

101 +

102 + # No `EngineClient` exists for the render server, so plugins get `None` and

103 + # must handle it themselves (see `EndpointPlugin.init_state`).

104 + await init_endpoint_plugins_state(None, state, args)

vllm/entrypoints/launchers/render/entry.py new

+120

@@ -0,0 +1,120 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import argparse

4 +import asyncio

5 +import signal

6 +import socket

7 +from argparse import Namespace

8 +

9 +from vllm import AsyncEngineArgs, envs

10 +from vllm.config import VllmConfig

11 +from vllm.logger import init_logger

12 +

13 +from ..app import build_app

14 +from ..launcher import serve_http, setup_server

15 +from ..utils.server_utils import get_uvicorn_log_config

16 +from .app_state import init_render_app_state

17 +

18 +logger = init_logger("vllm.entrypoints.launchers.render.entry")

19 +

20 +

21 +async def build_and_serve_renderer(

22 + vllm_config: VllmConfig,

23 + listen_address: str,

24 + sock: socket.socket,

25 + args: Namespace,

26 + **uvicorn_kwargs,

27 +) -> asyncio.Task:

28 + """Build FastAPI app for a CPU-only render server, initialize state, and

29 + start serving.

30 +

31 + Returns the shutdown task for the caller to await.

32 + """

33 +

34 + # Get uvicorn log config (from file or with endpoint filter)

35 + log_config = get_uvicorn_log_config(args)

36 + if log_config is not None:

37 + uvicorn_kwargs["log_config"] = log_config

38 +

39 + app = build_app(args, ("render",))

40 + await init_render_app_state(vllm_config, app.state, args)

41 +

42 + logger.info("Starting vLLM server on %s", listen_address)

43 +

44 + return await serve_http(

45 + app,

46 + sock=sock,

47 + enable_ssl_refresh=args.enable_ssl_refresh,

48 + host=args.host,

49 + port=args.port,

50 + log_level=args.uvicorn_log_level,

51 + # NOTE: When the 'disable_uvicorn_access_log' value is True,

52 + # no access log will be output.

53 + access_log=not args.disable_uvicorn_access_log,

54 + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,

55 + ssl_keyfile=args.ssl_keyfile,

56 + ssl_certfile=args.ssl_certfile,

57 + ssl_ca_certs=args.ssl_ca_certs,

58 + ssl_cert_reqs=args.ssl_cert_reqs,

59 + ssl_ciphers=args.ssl_ciphers,

60 + h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,

61 + h11_max_header_count=args.h11_max_header_count,

62 + **uvicorn_kwargs,

63 + )

64 +

65 +

66 +async def run_launch_fastapi(args: argparse.Namespace) -> None:

67 + """Run the online serving layer with FastAPI (no GPU inference)."""

68 +

69 + # Interrupt initialization if SIGTERM arrives before uvicorn installs

70 + # its own signal handlers. Once uvicorn is running it replaces this.

71 + def _interrupt_init(*_) -> None:

72 + raise KeyboardInterrupt("terminated")

73 +

74 + signal.signal(signal.SIGTERM, _interrupt_init)

75 +

76 + # 1. Socket binding

77 + listen_address, sock = setup_server(args, reuse_port=False)

78 +

79 + # 2. Build and serve the API server

80 + engine_args = AsyncEngineArgs.from_cli_args(args)

81 + model_config = engine_args.create_model_config()

82 +

83 + # Render servers preprocess data only — no inference, no quantized kernels.

84 + # Clear quantization so VllmConfig skips quant dtype/capability validation.

85 + model_config.quantization = None

86 +

87 + # Render servers never allocate KV cache; suppress the spurious CPU KV

88 + # cache space warning from CpuPlatform.check_and_update_config.

89 + envs.VLLM_CPU_KVCACHE_SPACE = 0

90 +

91 + vllm_config = VllmConfig(model_config=model_config)

92 + shutdown_task = await build_and_serve_renderer(

93 + vllm_config, listen_address, sock, args

94 + )

95 + try:

96 + await shutdown_task

97 + finally:

98 + sock.close()

99 +

100 +

101 +if __name__ == "__main__":

102 + import uvloop

103 +

104 + from vllm.entrypoints.openai.cli_args import (

105 + make_arg_parser,

106 + validate_parsed_serve_args,

107 + )

108 + from vllm.entrypoints.serve.utils.api_utils import cli_env_setup

109 + from vllm.utils.argparse_utils import FlexibleArgumentParser

110 +

111 + cli_env_setup()

112 + parser = FlexibleArgumentParser(

113 + description="Starts a GPU-less rendering server "

114 + "for preprocessing and postprocessing only"

115 + )

116 + parser = make_arg_parser(parser)

117 + args = parser.parse_args()

118 + validate_parsed_serve_args(args)

119 +

120 + uvloop.run(run_launch_fastapi(args))

vllm/entrypoints/launchers/utils/__init__.py

vllm/entrypoints/launchers/utils/constants.py renamed

-5

@@ -1,12 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""

4 -Shared constants for vLLM entrypoints.

5 -"""

4 # HTTP header limits for h11 parser

5 # These constants help mitigate header abuse attacks

6 H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB

7 H11_MAX_HEADER_COUNT_DEFAULT = 256

11 -

12 -MCP_PREFIX = "mcp_"

vllm/entrypoints/launchers/utils/server_utils.py renamed

+1 -1

@@ -12,7 +12,7 @@ from vllm.engine.protocol import EngineClient

12 from vllm.logger import init_logger

13 from vllm.utils.gc_utils import freeze_gc_heap

15 -logger = init_logger("vllm.entrypoints.openai.server_utils")

15 +logger = init_logger(__name__)

18 def load_log_config(log_config_file: str | None) -> dict | None:

vllm/entrypoints/launchers/utils/ssl.py renamed

vllm/entrypoints/openai/api_server.py

+46 -657

@@ -1,670 +1,59 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -import asyncio

4 -import multiprocessing

5 -import multiprocessing.forkserver as forkserver

6 -import os

7 -import signal

8 -import socket

9 -import tempfile

3 import warnings

11 -from argparse import Namespace

12 -from collections.abc import AsyncIterator

13 -from contextlib import asynccontextmanager

14 -from typing import Any, cast

16 -import uvloop

17 -from fastapi import FastAPI

18 -from starlette.datastructures import State

19 -

20 -import vllm.envs as envs

21 -from vllm.config import ModelConfig, VllmConfig

22 -from vllm.engine.arg_utils import AsyncEngineArgs

23 -from vllm.engine.protocol import EngineClient

24 -from vllm.entrypoints.chat_utils import load_chat_template

25 -from vllm.entrypoints.launcher import serve_http

5 +from vllm.entrypoints.launchers.api_server.app_state import init_app_state

6 +from vllm.entrypoints.launchers.api_server.entry import (

7 + build_and_serve,

8 + build_async_engine_client,

9 + build_async_engine_client_from_engine_args,

10 + run_server,

11 + run_server_worker,

12 +)

13 from vllm.entrypoints.launchers.api_server.routers import register_api_routers

27 -from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args

28 -from vllm.entrypoints.openai.models.protocol import BaseModelPath

29 -from vllm.entrypoints.openai.models.serving import OpenAIServingModels

30 -from vllm.entrypoints.serve.exception_handling.register import init_exception_handler

31 -from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware

32 -from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap

33 -from vllm.entrypoints.serve.tokenize.serving import ServingTokenization

34 -from vllm.entrypoints.serve.utils.api_utils import (

35 - cli_env_setup,

36 - log_non_default_args,

37 - log_version_and_model,

38 - process_lora_modules,

14 +from vllm.entrypoints.launchers.app import build_app

15 +from vllm.entrypoints.launchers.launcher import (

16 + create_server_socket,

17 + create_server_unix_socket,

18 + setup_server,

19 + validate_api_server_args,

20 )

40 -from vllm.entrypoints.serve.utils.request_logger import RequestLogger

41 -from vllm.entrypoints.serve.utils.server_utils import (

42 - get_uvicorn_log_config,

43 - lifespan,

21 +from vllm.entrypoints.launchers.render.app_state import init_render_app_state

22 +from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer

23 +

24 +warnings.warn(

25 + "`vllm.entrypoints.openai.api_server` is deprecated and will likely be"

26 + "unsupported in a future version. Use the corresponding function from "

27 + "`vllm.entrypoints.launchers` instead.",

28 + DeprecationWarning,

29 + stacklevel=1,

30 )

45 -from vllm.logger import init_logger

46 -from vllm.reasoning import ReasoningParserManager

47 -from vllm.renderers.online_derenderer import OnlineDerenderer

48 -from vllm.renderers.online_renderer import OnlineRenderer

49 -from vllm.tasks import POOLING_TASKS, SupportedTask

50 -from vllm.tool_parsers import ToolParserManager

51 -from vllm.tracing import instrument

52 -from vllm.usage.usage_lib import UsageContext

53 -from vllm.utils.argparse_utils import FlexibleArgumentParser

54 -from vllm.utils.network_utils import is_valid_ipv6_address

55 -from vllm.utils.system_utils import decorate_logs, set_ulimit

56 -from vllm.version import __version__ as VLLM_VERSION

57 -

58 -prometheus_multiproc_dir: tempfile.TemporaryDirectory

59 -

60 -# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)

61 -logger = init_logger("vllm.entrypoints.openai.api_server")

62 -

63 -_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)

64 -

65 -

66 -def _attach_endpoint_plugins(

67 - app: FastAPI, supported_tasks: tuple["SupportedTask", ...]

68 -) -> None:

69 - """Phase A of endpoint plugin wiring: discover, gate and attach routes.

70 -

71 - Attached last after all core routers. This is so endpoint plugin routes can

72 - shadow core routes with the same path (see `EndpointPlugin.attach_router`

73 - docstring). No-ops when no plugins are discovered/allowlisted.

74 - """

75 - from vllm.plugins import load_endpoint_plugins

76 -

77 - endpoint_plugins = load_endpoint_plugins(supported_tasks)

78 - for plugin in endpoint_plugins:

79 - plugin.attach_router(app)

80 - app.state.endpoint_plugins = endpoint_plugins

81 -

82 -

83 -async def _init_endpoint_plugins_state(

84 - engine_client: EngineClient | None, state: State, args: Namespace

85 -) -> None:

86 - """Phase B of endpoint plugin wiring: initialize per app plugin state.

87 -

88 - `state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)

89 - in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)

90 - build their own bare `State` without going through `build_app`. As a result

91 - `endpoint_plugins` may be absent and are treated that the same as "none attached".

92 -

93 - `engine_client` is `None` for the CPU only render server which has no

94 - engine (see `init_render_app_state`). Plugins must handle a `None`

95 - `engine_client` themselves (see `EndpointPlugin.init_state`).

96 - """

97 - for plugin in getattr(state, "endpoint_plugins", []):

98 - await plugin.init_state(engine_client, state, args)

99 -

100 -

101 -@asynccontextmanager

102 -async def build_async_engine_client(

103 - args: Namespace,

104 - *,

105 - usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,

106 - client_config: dict[str, Any] | None = None,

107 -) -> AsyncIterator[EngineClient]:

108 - if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":

109 - # The executor is expected to be mp.

110 - # Pre-import heavy modules in the forkserver process

111 - logger.debug("Setup forkserver with pre-imports")

112 - multiprocessing.set_start_method("forkserver")

113 - multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])

114 - forkserver.ensure_running()

115 - logger.debug("Forkserver setup complete!")

116 -

117 - # Context manager to handle engine_client lifecycle

118 - # Ensures everything is shutdown and cleaned up on error/exit

119 - engine_args = AsyncEngineArgs.from_cli_args(args)

120 - if client_config:

121 - engine_args._api_process_count = client_config.get("client_count", 1)

122 - engine_args._api_process_rank = client_config.get("client_index", 0)

123 -

124 - async with build_async_engine_client_from_engine_args(

125 - engine_args,

126 - usage_context=usage_context,

127 - client_config=client_config,

128 - ) as engine:

129 - yield engine

130 -

131 -

132 -@asynccontextmanager

133 -async def build_async_engine_client_from_engine_args(

134 - engine_args: AsyncEngineArgs,

135 - *,

136 - usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,

137 - client_config: dict[str, Any] | None = None,

138 -) -> AsyncIterator[EngineClient]:

139 - """

140 - Create EngineClient, either:

141 - - in-process using the AsyncLLMEngine Directly

142 - - multiprocess using AsyncLLMEngine RPC

143 -

144 - Returns the Client or None if the creation failed.

145 - """

146 -

147 - # Create the EngineConfig (determines if we can use V1).

148 - vllm_config = engine_args.create_engine_config(usage_context=usage_context)

149 -

150 - from vllm.v1.engine.async_llm import AsyncLLM

151 -

152 - async_llm: AsyncLLM | None = None

153 -

154 - # Don't mutate the input client_config

155 - client_config = dict(client_config) if client_config else {}

156 - client_count = client_config.pop("client_count", 1)

157 - client_index = client_config.pop("client_index", 0)

158 -

159 - try:

160 - async_llm = AsyncLLM.from_vllm_config(

161 - vllm_config=vllm_config,

162 - usage_context=usage_context,

163 - enable_log_requests=engine_args.enable_log_requests,

164 - aggregate_engine_logging=engine_args.aggregate_engine_logging,

165 - disable_log_stats=engine_args.disable_log_stats,

166 - client_addresses=client_config,

167 - client_count=client_count,

168 - client_index=client_index,

169 - )

170 -

171 - # Don't keep the dummy data in memory

172 - assert async_llm is not None

173 - await async_llm.reset_mm_cache()

174 -

175 - yield async_llm

176 - finally:

177 - if async_llm:

178 - async_llm.shutdown(timeout=vllm_config.shutdown_timeout)

179 -

180 -

181 -def build_app(

182 - args: Namespace,

183 - supported_tasks: tuple["SupportedTask", ...] | None = None,

184 - model_config: ModelConfig | None = None,

185 -) -> FastAPI:

186 - if supported_tasks is None:

187 - warnings.warn(

188 - "The 'supported_tasks' parameter was not provided to "

189 - "build_app and will be required in a future version. "

190 - "Defaulting to ('generate',).",

191 - DeprecationWarning,

192 - stacklevel=2,

193 - )

194 - supported_tasks = _FALLBACK_SUPPORTED_TASKS

195 -

196 - if args.disable_fastapi_docs:

197 - app = FastAPI(

198 - openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan

199 - )

200 - elif args.enable_offline_docs:

201 - app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)

202 - else:

203 - app = FastAPI(lifespan=lifespan)

204 - app.state.args = args

205 - app.root_path = args.root_path

206 -

207 - register_api_routers(args, app, supported_tasks, model_config)

208 -

209 - # Endpoint plugins are attached last so their routes are registered after all core

210 - # routers. This runs even for the CPU only render server. A plugin eligible for

211 - # the `render` task still gets its routes registered. It receives

212 - # `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).

213 - _attach_endpoint_plugins(app, supported_tasks)

214 -

215 - init_exception_handler(app)

216 - init_entrypoints_middleware(args, app, supported_tasks)

217 - app = sagemaker_standards_bootstrap(app)

218 - return app

219 -

220 -

221 -async def init_app_state(

222 - engine_client: EngineClient,

223 - state: State,

224 - args: Namespace,

225 - supported_tasks: tuple["SupportedTask", ...] | None = None,

226 -) -> None:

227 - vllm_config = engine_client.vllm_config

228 -

229 - if args.tool_call_parser is not None:

230 - from vllm.parser.metrics import init_parser_metrics

231 -

232 - init_parser_metrics(

233 - model_name=cast(str, vllm_config.model_config.served_model_name)

234 - )

235 -

236 - if supported_tasks is None:

237 - warnings.warn(

238 - "The 'supported_tasks' parameter was not provided to "

239 - "init_app_state and will be required in a future version. "

240 - "Please pass 'supported_tasks' explicitly.",

241 - DeprecationWarning,

242 - stacklevel=2,

243 - )

244 - supported_tasks = _FALLBACK_SUPPORTED_TASKS

245 -

246 - if args.served_model_name is not None:

247 - served_model_names = args.served_model_name

248 - else:

249 - served_model_names = [args.model]

250 -

251 - if args.enable_log_requests:

252 - request_logger = RequestLogger(max_log_len=args.max_log_len)

253 - else:

254 - request_logger = None

255 -

256 - base_model_paths = [

257 - BaseModelPath(name=name, model_path=args.model) for name in served_model_names

258 - ]

259 -

260 - state.engine_client = engine_client

261 - state.log_stats = not args.disable_log_stats

262 - state.vllm_config = vllm_config

263 - state.args = args

264 - resolved_chat_template = load_chat_template(args.chat_template)

265 -

266 - # Merge default_mm_loras into the static lora_modules

267 - default_mm_loras = (

268 - vllm_config.lora_config.default_mm_loras

269 - if vllm_config.lora_config is not None

270 - else {}

271 - )

272 - lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)

273 -

274 - state.openai_serving_models = OpenAIServingModels(

275 - engine_client=engine_client,

276 - base_model_paths=base_model_paths,

277 - lora_modules=lora_modules,

278 - )

279 - await state.openai_serving_models.init_static_loras()

280 -

281 - state.online_renderer = OnlineRenderer(

282 - model_config=engine_client.model_config,

283 - renderer=engine_client.renderer,

284 - request_logger=request_logger,

285 - chat_template=resolved_chat_template,

286 - chat_template_content_format=args.chat_template_content_format,

287 - trust_request_chat_template=args.trust_request_chat_template,

288 - enable_auto_tools=args.enable_auto_tool_choice,

289 - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

290 - tool_parser=args.tool_call_parser,

291 - reasoning_parser=args.structured_outputs_config.reasoning_parser,

292 - default_chat_template_kwargs=args.default_chat_template_kwargs,

293 - log_error_stack=args.log_error_stack,

294 - )

295 - state.online_renderer.warmup()

296 -

297 - state.online_derenderer = OnlineDerenderer(

298 - model_config=engine_client.model_config,

299 - renderer=engine_client.renderer,

300 - request_logger=request_logger,

301 - chat_template=resolved_chat_template,

302 - chat_template_content_format=args.chat_template_content_format,

303 - trust_request_chat_template=args.trust_request_chat_template,

304 - enable_auto_tools=args.enable_auto_tool_choice,

305 - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

306 - tool_parser=args.tool_call_parser,

307 - reasoning_parser=args.structured_outputs_config.reasoning_parser,

308 - default_chat_template_kwargs=args.default_chat_template_kwargs,

309 - log_error_stack=args.log_error_stack,

310 - )

311 -

312 - state.serving_tokenization = ServingTokenization(

313 - state.openai_serving_models,

314 - state.online_renderer,

315 - request_logger=request_logger,

316 - chat_template=resolved_chat_template,

317 - chat_template_content_format=args.chat_template_content_format,

318 - default_chat_template_kwargs=args.default_chat_template_kwargs,

319 - trust_request_chat_template=args.trust_request_chat_template,

320 - )

321 -

322 - if "generate" in supported_tasks:

323 - from vllm.entrypoints.generate.api_router import init_generate_state

324 -

325 - await init_generate_state(

326 - engine_client, state, args, request_logger, supported_tasks

327 - )

328 -

329 - from vllm.entrypoints.scale_out.factories import init_scale_out_state

330 -

331 - init_scale_out_state(state, args, engine_client, request_logger)

332 -

333 - if "transcription" in supported_tasks or "realtime" in supported_tasks:

334 - from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state

335 -

336 - init_speech_to_text_state(

337 - engine_client, state, args, request_logger, supported_tasks

338 - )

339 -

340 - if any(task in POOLING_TASKS for task in supported_tasks):

341 - from vllm.entrypoints.pooling.factories import init_pooling_state

342 -

343 - init_pooling_state(engine_client, state, args, request_logger, supported_tasks)

344 -

345 - await _init_endpoint_plugins_state(engine_client, state, args)

346 -

347 - state.enable_server_load_tracking = args.enable_server_load_tracking

348 - state.server_load_metrics = 0

349 -

350 -

351 -async def init_render_app_state(

352 - vllm_config: VllmConfig,

353 - state: State,

354 - args: Namespace,

355 -) -> None:

356 - """Initialise FastAPI app state for a CPU-only render server.

357 -

358 - Unlike :func:`init_app_state` this function does not require an

359 - :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the

360 - preprocessing pipeline (renderer, input_processor)

361 - directly from the :class:`~vllm.config.VllmConfig`.

362 - """

363 - from vllm.entrypoints.chat_utils import load_chat_template

364 - from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry

365 - from vllm.renderers import renderer_from_config

366 - from vllm.renderers.online_renderer import OnlineRenderer

367 -

368 - served_model_names = args.served_model_name or [args.model]

369 - model_registry = OpenAIModelRegistry(

370 - model_config=vllm_config.model_config,

371 - base_model_paths=[

372 - BaseModelPath(name=name, model_path=args.model)

373 - for name in served_model_names

374 - ],

375 - )

376 -

377 - if args.enable_log_requests:

378 - request_logger = RequestLogger(max_log_len=args.max_log_len)

379 - else:

380 - request_logger = None

381 -

382 - renderer = renderer_from_config(vllm_config)

383 - resolved_chat_template = load_chat_template(args.chat_template)

384 -

385 - state.online_renderer = OnlineRenderer(

386 - model_config=vllm_config.model_config,

387 - renderer=renderer,

388 - request_logger=request_logger,

389 - chat_template=resolved_chat_template,

390 - chat_template_content_format=args.chat_template_content_format,

391 - trust_request_chat_template=args.trust_request_chat_template,

392 - enable_auto_tools=args.enable_auto_tool_choice,

393 - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

394 - tool_parser=args.tool_call_parser,

395 - reasoning_parser=args.reasoning_parser,

396 - default_chat_template_kwargs=args.default_chat_template_kwargs,

397 - log_error_stack=args.log_error_stack,

398 - )

399 - state.online_renderer.warmup()

400 -

401 - state.online_derenderer = OnlineDerenderer(

402 - model_config=vllm_config.model_config,

403 - renderer=renderer,

404 - request_logger=request_logger,

405 - chat_template=resolved_chat_template,

406 - chat_template_content_format=args.chat_template_content_format,

407 - trust_request_chat_template=args.trust_request_chat_template,

408 - enable_auto_tools=args.enable_auto_tool_choice,

409 - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

410 - tool_parser=args.tool_call_parser,

411 - reasoning_parser=args.reasoning_parser,

412 - default_chat_template_kwargs=args.default_chat_template_kwargs,

413 - log_error_stack=args.log_error_stack,

414 - )

415 -

416 - state.openai_serving_models = model_registry

417 - state.serving_tokenization = ServingTokenization(

418 - model_registry,

419 - state.online_renderer,

420 - request_logger=request_logger,

421 - chat_template=resolved_chat_template,

422 - chat_template_content_format=args.chat_template_content_format,

423 - default_chat_template_kwargs=args.default_chat_template_kwargs,

424 - trust_request_chat_template=args.trust_request_chat_template,

425 - )

426 -

427 - from vllm.entrypoints.scale_out.factories import init_render_state

428 -

429 - init_render_state(state, request_logger)

430 -

431 - state.vllm_config = vllm_config

432 - # Disable stats logging — there is no engine to poll.

433 - state.log_stats = False

434 - state.engine_client = None

435 - state.args = args

436 - state.enable_server_load_tracking = False

437 - state.server_load_metrics = 0

438 -

439 - # No `EngineClient` exists for the render server, so plugins get `None` and

440 - # must handle it themselves (see `EndpointPlugin.init_state`).

441 - await _init_endpoint_plugins_state(None, state, args)

442 -

443 -

444 -def create_server_socket(

445 - addr: tuple[str, int],

446 - *,

447 - reuse_port: bool,

448 -) -> socket.socket:

449 - family = socket.AF_INET

450 - if is_valid_ipv6_address(addr[0]):

451 - family = socket.AF_INET6

452 -

453 - sock = socket.socket(family=family, type=socket.SOCK_STREAM)

454 - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

455 - if reuse_port:

456 - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)

457 - sock.bind(addr)

458 -

459 - return sock

460 -

461 -

462 -def create_server_unix_socket(path: str) -> socket.socket:

463 - sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)

464 - sock.bind(path)

465 - return sock

466 -

467 -

468 -def validate_api_server_args(args):

469 - valid_tool_parses = ToolParserManager.list_registered()

470 - if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:

471 - raise KeyError(

472 - f"invalid tool call parser: {args.tool_call_parser} "

473 - f"(chose from {{ {','.join(valid_tool_parses)} }})"

474 - )

475 -

476 - valid_reasoning_parsers = ReasoningParserManager.list_registered()

477 - if (

478 - reasoning_parser := args.structured_outputs_config.reasoning_parser

479 - ) and reasoning_parser not in valid_reasoning_parsers:

480 - raise KeyError(

481 - f"invalid reasoning parser: {reasoning_parser} "

482 - f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"

483 - )

484 -

485 -

486 -@instrument(span_name="API server setup")

487 -def setup_server(args, *, reuse_port: bool):

488 - """Validate API server args and create the server socket."""

489 -

490 - log_version_and_model(logger, VLLM_VERSION, args.model)

491 - log_non_default_args(args)

492 -

493 - if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:

494 - ToolParserManager.import_tool_parser(args.tool_parser_plugin)

495 -

496 - if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:

497 - ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

498 -

499 - validate_api_server_args(args)

500 -

501 - # workaround to make sure that we bind the port before the engine is set up.

502 - # This avoids race conditions with ray.

503 - # see https://github.com/vllm-project/vllm/issues/8204

504 - if args.uds:

505 - sock = create_server_unix_socket(args.uds)

506 - else:

507 - sock_addr = (args.host or "", args.port)

508 - sock = create_server_socket(sock_addr, reuse_port=reuse_port)

509 -

510 - # workaround to avoid footguns where uvicorn drops requests with too

511 - # many concurrent requests active

512 - set_ulimit()

513 -

514 - if args.uds:

515 - listen_address = f"unix:{args.uds}"

516 - else:

517 - addr, port = sock_addr

518 - is_ssl = args.ssl_keyfile and args.ssl_certfile

519 - host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"

520 - listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"

521 - return listen_address, sock

522 -

523 -

524 -async def build_and_serve(

525 - engine_client: EngineClient,

526 - listen_address: str,

527 - sock: socket.socket,

528 - args: Namespace,

529 - **uvicorn_kwargs,

530 -) -> asyncio.Task:

531 - """Build FastAPI app, initialize state, and start serving.

532 -

533 - Returns the shutdown task for the caller to await.

534 - """

535 -

536 - # Get uvicorn log config (from file or with endpoint filter)

537 - log_config = get_uvicorn_log_config(args)

538 - if log_config is not None:

539 - uvicorn_kwargs["log_config"] = log_config

540 -

541 - supported_tasks = await engine_client.get_supported_tasks()

542 - model_config = engine_client.model_config

543 -

544 - logger.info("Supported tasks: %s", supported_tasks)

545 - app = build_app(args, supported_tasks, model_config)

546 - await init_app_state(engine_client, app.state, args, supported_tasks)

547 -

548 - logger.info("Starting vLLM server on %s", listen_address)

549 -

550 - return await serve_http(

551 - app,

552 - sock=sock,

553 - enable_ssl_refresh=args.enable_ssl_refresh,

554 - host=args.host,

555 - port=args.port,

556 - log_level=args.uvicorn_log_level,

557 - # NOTE: When the 'disable_uvicorn_access_log' value is True,

558 - # no access log will be output.

559 - access_log=not args.disable_uvicorn_access_log,

560 - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,

561 - ssl_keyfile=args.ssl_keyfile,

562 - ssl_certfile=args.ssl_certfile,

563 - ssl_ca_certs=args.ssl_ca_certs,

564 - ssl_cert_reqs=args.ssl_cert_reqs,

565 - ssl_ciphers=args.ssl_ciphers,

566 - h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,

567 - h11_max_header_count=args.h11_max_header_count,

568 - **uvicorn_kwargs,

569 - )

570 -

571 -

572 -async def build_and_serve_renderer(

573 - vllm_config: VllmConfig,

574 - listen_address: str,

575 - sock: socket.socket,

576 - args: Namespace,

577 - **uvicorn_kwargs,

578 -) -> asyncio.Task:

579 - """Build FastAPI app for a CPU-only render server, initialize state, and

580 - start serving.

581 -

582 - Returns the shutdown task for the caller to await.

583 - """

584 -

585 - # Get uvicorn log config (from file or with endpoint filter)

586 - log_config = get_uvicorn_log_config(args)

587 - if log_config is not None:

588 - uvicorn_kwargs["log_config"] = log_config

589 -

590 - app = build_app(args, ("render",))

591 - await init_render_app_state(vllm_config, app.state, args)

592 -

593 - logger.info("Starting vLLM server on %s", listen_address)

594 -

595 - return await serve_http(

596 - app,

597 - sock=sock,

598 - enable_ssl_refresh=args.enable_ssl_refresh,

599 - host=args.host,

600 - port=args.port,

601 - log_level=args.uvicorn_log_level,

602 - # NOTE: When the 'disable_uvicorn_access_log' value is True,

603 - # no access log will be output.

604 - access_log=not args.disable_uvicorn_access_log,

605 - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,

606 - ssl_keyfile=args.ssl_keyfile,

607 - ssl_certfile=args.ssl_certfile,

608 - ssl_ca_certs=args.ssl_ca_certs,

609 - ssl_cert_reqs=args.ssl_cert_reqs,

610 - ssl_ciphers=args.ssl_ciphers,

611 - h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,

612 - h11_max_header_count=args.h11_max_header_count,

613 - **uvicorn_kwargs,

614 - )

615 -

616 -

617 -async def run_server(args, **uvicorn_kwargs) -> None:

618 - """Run a single-worker API server."""

619 -

620 - decorate_logs("APIServer", skip_if_decorated=True)

621 -

622 - # Interrupt initialization if SIGTERM arrives before uvicorn installs its

623 - # own signal handlers. Once uvicorn is running it replaces this.

624 - def _interrupt_init(*_) -> None:

625 - raise KeyboardInterrupt("terminated")

626 -

627 - signal.signal(signal.SIGTERM, _interrupt_init)

628 -

629 - listen_address, sock = setup_server(args, reuse_port=False)

630 - await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)

631 -

632 -

633 -async def run_server_worker(

634 - listen_address, sock, args, client_config=None, **uvicorn_kwargs

635 -) -> None:

636 - """Run a single API server worker."""

637 -

638 - if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:

639 - ToolParserManager.import_tool_parser(args.tool_parser_plugin)

640 -

641 - if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:

642 - ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

644 - async with build_async_engine_client(

645 - args,

646 - client_config=client_config,

647 - ) as engine_client:

648 - shutdown_task = await build_and_serve(

649 - engine_client, listen_address, sock, args, **uvicorn_kwargs

650 - )

651 - # NB: Await server shutdown only after the backend context is exited

652 - try:

653 - await shutdown_task

654 - finally:

655 - sock.close()

33 +__all__ = [

34 + "build_async_engine_client",

35 + "build_async_engine_client_from_engine_args",

36 + "build_app",

37 + "init_app_state",

38 + "init_render_app_state",

39 + "create_server_socket",

40 + "create_server_unix_socket",

41 + "validate_api_server_args",

42 + "setup_server",

43 + "build_and_serve",

44 + "build_and_serve_renderer",

45 + "run_server",

46 + "run_server_worker",

47 + "register_api_routers",

48 +]

50 if __name__ == "__main__":

659 - # NOTE(simon):

660 - # This section should be in sync with vllm/entrypoints/cli/main.py for CLI

661 - # entrypoints.

662 - cli_env_setup()

663 - parser = FlexibleArgumentParser(

664 - description="vLLM OpenAI-Compatible RESTful API server."

51 + warnings.warn(

52 + "The `python -m vllm.entrypoints.openai.api_server` command is deprecated "

53 + "and may be removed in a future release. Please use `vllm server` instead.",

54 + DeprecationWarning,

55 + stacklevel=1,

56 )

666 - parser = make_arg_parser(parser)

667 - args = parser.parse_args()

668 - validate_parsed_serve_args(args)

57 + from vllm.entrypoints.launchers.api_server.entry import main

670 - uvloop.run(run_server(args))

59 + main()

vllm/entrypoints/openai/cli_args.py

+3 -3

@@ -20,11 +20,11 @@ from vllm.entrypoints.chat_utils import (

20 ChatTemplateContentFormatOption,

21 validate_chat_template,

22 )

23 -from vllm.entrypoints.openai.models.protocol import LoRAModulePath

24 -from vllm.entrypoints.serve.utils.constants import (

23 +from vllm.entrypoints.launchers.utils.constants import (

24 H11_MAX_HEADER_COUNT_DEFAULT,

25 H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,

26 )

27 +from vllm.entrypoints.openai.models.protocol import LoRAModulePath

28 from vllm.tool_parsers import ToolParserManager

29 from vllm.utils.argparse_utils import FlexibleArgumentParser

@@ -444,6 +444,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):

444

445 def create_parser_for_docs() -> FlexibleArgumentParser:

446 parser_for_docs = FlexibleArgumentParser(

447 - prog="-m vllm.entrypoints.openai.api_server"

447 + prog="-m vllm.entrypoints.launchers.api_server.entry"

448 )

449 return make_arg_parser(parser_for_docs)

vllm/entrypoints/openai/dp_supervisor.py

+2 -2

@@ -23,7 +23,7 @@ import uvloop

23 from fastapi import FastAPI, Response

25 import vllm.envs as envs

26 -from vllm.entrypoints.launcher import NoSignalServer

26 +from vllm.entrypoints.launchers.launcher import NoSignalServer

27 from vllm.logger import init_logger

28 from vllm.utils.system_utils import (

29 decorate_logs,

@@ -236,7 +236,7 @@ def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:

236

237

238 def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:

239 - from vllm.entrypoints.openai.api_server import run_server

239 + from vllm.entrypoints.launchers.api_server.entry import run_server

240

241 uvloop.run(run_server(child_args))

242

vllm/entrypoints/openai/responses/context.py

+1 -1

@@ -39,7 +39,6 @@ from vllm.entrypoints.openai.responses.utils import (

39 build_response_output_items,

40 construct_tool_dicts,

41 )

42 -from vllm.entrypoints.serve.utils.constants import MCP_PREFIX

42 from vllm.outputs import RequestOutput

43 from vllm.parser.abstract_parser import Parser

44 from vllm.tokenizers import TokenizerLike

@@ -58,6 +57,7 @@ _TOOL_NAME_TO_TYPE_MAP = {

57 "python": "code_interpreter",

58 "container": "container",

59 }

60 +MCP_PREFIX = "mcp_"

63 def _map_tool_name_to_tool_type(tool_name: str) -> str:

vllm/entrypoints/openai/run_batch.py

+3 -4

@@ -31,7 +31,7 @@ from vllm.config import config

31 from vllm.connections import global_http_connection

32 from vllm.engine.arg_utils import AsyncEngineArgs

33 from vllm.engine.protocol import EngineClient

34 -from vllm.entrypoints.openai.api_server import init_app_state

34 +from vllm.entrypoints.launchers.api_server.entry import init_app_state

35 from vllm.entrypoints.openai.chat_completion.protocol import (

36 ChatCompletionRequest,

37 ChatCompletionResponse,

@@ -836,8 +836,7 @@ async def run_batch(

836 error_msg=f"URL {request.url} was used. "

837 "Supported endpoints: /v1/chat/completions, /v1/embeddings,"

838 " /v1/audio/transcriptions, /v1/audio/translations, /score, "

839 - " /rerank. See vllm/entrypoints/openai/api_server.py "

840 - "for supported score/rerank versions.",

839 + " /rerank.",

840 )

841 )

842

@@ -848,7 +847,7 @@ async def run_batch(

847

848

849 async def main(args: Namespace):

851 - from vllm.entrypoints.openai.api_server import build_async_engine_client

850 + from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client

851 from vllm.usage.usage_lib import UsageContext

852

853 validate_run_batch_args(args)

vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py

+1 -1

@@ -4,7 +4,7 @@

4 from fastapi import Request

5 from starlette.responses import JSONResponse

7 -from vllm.entrypoints.launcher import terminate_if_errored

7 +from vllm.entrypoints.launchers.launcher import terminate_if_errored

8 from vllm.entrypoints.openai.engine.protocol import GenerationError

9 from vllm.exceptions import VLLMError

10 from vllm.logger import init_logger

vllm/plugins/endpoint_plugins/interface.py

+38 -2

@@ -20,7 +20,7 @@ for the loading/gating rules and `docs/usage/security.md` for the security

20 posture of exposing plugin defined routes.

22 The CPU only render server (see `build_and_serve_renderer` in

23 -`vllm/entrypoints/openai/api_server.py`) has no `EngineClient`. A plugin

23 +`vllm/entrypoints/launchers/render`) has no `EngineClient`. A plugin

24 eligible for the `render` task (`required_tasks` is `None` or includes

25 `"render"`) still gets `attach_router` called but `init_state` receives

26 `engine_client=None`. Plugins that cannot function without an engine should

@@ -34,8 +34,9 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable

34 from fastapi import FastAPI

35 from starlette.datastructures import State

37 +from vllm.engine.protocol import EngineClient

38 +

39 if TYPE_CHECKING:

38 - from vllm.engine.protocol import EngineClient

40 from vllm.tasks import SupportedTask

@@ -85,3 +86,38 @@ class EndpointPlugin(Protocol):

86 cannot function without an engine.

87 """

88 ...

89 +

90 +

91 +def attach_endpoint_plugins(

92 + app: FastAPI, supported_tasks: tuple["SupportedTask", ...]

93 +) -> None:

94 + """Phase A of endpoint plugin wiring: discover, gate and attach routes.

95 +

96 + Attached last after all core routers. This is so endpoint plugin routes can

97 + shadow core routes with the same path (see `EndpointPlugin.attach_router`

98 + docstring). No-ops when no plugins are discovered/allowlisted.

99 + """

100 + from vllm.plugins import load_endpoint_plugins

101 +

102 + endpoint_plugins = load_endpoint_plugins(supported_tasks)

103 + for plugin in endpoint_plugins:

104 + plugin.attach_router(app)

105 + app.state.endpoint_plugins = endpoint_plugins

106 +

107 +

108 +async def init_endpoint_plugins_state(

109 + engine_client: EngineClient | None, state: State, args: Namespace

110 +) -> None:

111 + """Phase B of endpoint plugin wiring: initialize per app plugin state.

112 +

113 + `state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)

114 + in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)

115 + build their own bare `State` without going through `build_app`. As a result

116 + `endpoint_plugins` may be absent and are treated that the same as "none attached".

117 +

118 + `engine_client` is `None` for the CPU only render server which has no

119 + engine (see `init_render_app_state`). Plugins must handle a `None`

120 + `engine_client` themselves (see `EndpointPlugin.init_state`).

121 + """

122 + for plugin in getattr(state, "endpoint_plugins", []):

123 + await plugin.init_state(engine_client, state, args)

vllm/tasks.py

@@ -41,3 +41,4 @@ FrontendTask = Literal["render"]

41 FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask)

43 SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask]

44 +FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)

vllm/v1/utils.py

+1 -1

@@ -503,7 +503,7 @@ def run_api_server_worker_proc(

503 ) -> None:

504 """Entrypoint for individual API server worker processes."""

505

506 - from vllm.entrypoints.openai.api_server import run_server_worker

506 + from vllm.entrypoints.launchers.api_server.entry import run_server_worker

507

508 client_config = client_config or {}

509 server_index = client_config.get("client_index", 0)