A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 07:30 UTC
[Refactor] [4/N] Move VLLM_SERVER_DEV endpoints into the serve directory (#30749)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Dec 17, 2025 at 18:27 UTC
9ad5b2171002522772de0a0cc71b747068ec8862
19 files changed
+259
-151
.buildkite/scripts/hardware_ci/run-amd-test.sh
-1
@@ -141,7 +141,6 @@ if [[ $commands == *" entrypoints/openai "* ]]; then
141
--ignore=entrypoints/openai/test_audio.py \
142
--ignore=entrypoints/openai/test_shutdown.py \
143
--ignore=entrypoints/openai/test_completion.py \
144
- --ignore=entrypoints/openai/test_sleep.py \
144
--ignore=entrypoints/openai/test_models.py \
145
--ignore=entrypoints/openai/test_lora_adapters.py \
146
--ignore=entrypoints/openai/test_return_tokens_as_ids.py \
.buildkite/test-amd.yaml
+22
-15
@@ -128,7 +128,7 @@ steps:
128
- tests/entrypoints/
129
commands:
130
- pytest -v -s entrypoints/openai/tool_parsers
131
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
131
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/instrumentator --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
132
133
- label: Entrypoints Integration Test (LLM) # 30min
134
timeout_in_minutes: 40
@@ -148,7 +148,7 @@ steps:
148
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
149
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
150
151
-- label: Entrypoints Integration Test (API Server) # 100min
151
+- label: Entrypoints Integration Test (API Server 1) # 100min
152
timeout_in_minutes: 130
153
mirror_hardwares: [amdexperimental]
154
agent_pool: mi325_1
@@ -162,10 +162,28 @@ steps:
162
- tests/entrypoints/test_chat_utils
163
commands:
164
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
165
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/openai/test_collective_rpc.py # PYTHONPATH is needed to import custom Worker extension
166
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_collective_rpc.py --ignore=entrypoints/openai/tool_parsers/
165
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/
166
- pytest -v -s entrypoints/test_chat_utils.py
167
168
+- label: Entrypoints Integration Test (API Server 2)
169
+ timeout_in_minutes: 50
170
+ mirror_hardwares: [amdexperimental]
171
+ agent_pool: mi325_1
172
+ # grade: Blocking
173
+ working_dir: "/vllm-workspace/tests"
174
+ fast_check: true
175
+ torch_nightly: true
176
+ source_file_dependencies:
177
+ - vllm/
178
+ - tests/entrypoints/sleep
179
+ - tests/entrypoints/rpc
180
+ - tests/tool_use
181
+ commands:
182
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
183
+ - pytest -v -s entrypoints/sleep
184
+ - pytest -v -s tool_use
185
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
186
+
187
- label: Entrypoints Integration Test (Pooling)
188
timeout_in_minutes: 50
189
mirror_hardwares: [amdexperimental]
@@ -751,17 +769,6 @@ steps:
769
# Transcription WER check is skipped because encoder-decoder models are not supported on ROCm, see https://github.com/vllm-project/vllm/issues/27442
770
- pytest -s entrypoints/openai/correctness/
771
754
-- label: OpenAI-Compatible Tool Use # 23 min
755
- timeout_in_minutes: 35
756
- mirror_hardwares: [amdexperimental, amdproduction]
757
- agent_pool: mi325_1
758
- # grade: Blocking
759
- fast_check: false
760
- source_file_dependencies:
761
- - vllm/
762
- - tests/tool_use
763
- commands:
764
- - pytest -v -s tool_use
772
773
##### models test #####
774
.buildkite/test-pipeline.yaml
+20
-14
@@ -114,7 +114,7 @@ steps:
114
- tests/entrypoints/
115
commands:
116
- pytest -v -s entrypoints/openai/tool_parsers
117
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
117
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
118
119
- label: Entrypoints Integration Test (LLM) # 30min
120
timeout_in_minutes: 40
@@ -132,7 +132,7 @@ steps:
132
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
133
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
134
135
-- label: Entrypoints Integration Test (API Server) # 100min
135
+- label: Entrypoints Integration Test (API Server 1) # 100min
136
timeout_in_minutes: 130
137
mirror_hardwares: [amdexperimental]
138
working_dir: "/vllm-workspace/tests"
@@ -144,10 +144,26 @@ steps:
144
- tests/entrypoints/test_chat_utils
145
commands:
146
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
147
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/openai/test_collective_rpc.py # PYTHONPATH is needed to import custom Worker extension
148
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_collective_rpc.py --ignore=entrypoints/openai/tool_parsers/
147
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/
148
- pytest -v -s entrypoints/test_chat_utils.py
149
150
+- label: Entrypoints Integration Test (API Server 2)
151
+ timeout_in_minutes: 50
152
+ mirror_hardwares: [amdexperimental]
153
+ working_dir: "/vllm-workspace/tests"
154
+ fast_check: true
155
+ torch_nightly: true
156
+ source_file_dependencies:
157
+ - vllm/
158
+ - tests/entrypoints/sleep
159
+ - tests/entrypoints/rpc
160
+ - tests/tool_use
161
+ commands:
162
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
163
+ - pytest -v -s entrypoints/sleep
164
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
165
+ - pytest -v -s tool_use
166
+
167
- label: Entrypoints Integration Test (Pooling)
168
timeout_in_minutes: 50
169
mirror_hardwares: [amdexperimental]
@@ -666,16 +682,6 @@ steps:
682
commands: # LMEval+Transcription WER check
683
- pytest -s entrypoints/openai/correctness/
684
669
-- label: OpenAI-Compatible Tool Use # 23 min
670
- timeout_in_minutes: 35
671
- mirror_hardwares: [amdexperimental]
672
- fast_check: false
673
- source_file_dependencies:
674
- - vllm/
675
- - tests/tool_use
676
- commands:
677
- - pytest -v -s tool_use
678
-
685
##### models test #####
686
687
- label: Basic Models Tests (Initialization)
.buildkite/test_areas/entrypoints.yaml
+19
-4
@@ -10,7 +10,7 @@ steps:
10
- tests/entrypoints/
11
commands:
12
- pytest -v -s entrypoints/openai/tool_parsers
13
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
13
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
14
15
- label: Entrypoints Integration (LLM)
16
timeout_in_minutes: 40
@@ -25,7 +25,7 @@ steps:
25
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
26
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
27
28
-- label: Entrypoints Integration (API Server)
28
+- label: Entrypoints Integration (API Server 1)
29
timeout_in_minutes: 130
30
working_dir: "/vllm-workspace/tests"
31
source_file_dependencies:
@@ -34,11 +34,26 @@ steps:
34
- tests/entrypoints/test_chat_utils
35
commands:
36
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
37
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/openai/test_collective_rpc.py # PYTHONPATH is needed to import custom Worker extension
38
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_collective_rpc.py --ignore=entrypoints/openai/tool_parsers/
37
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/
38
- pytest -v -s entrypoints/test_chat_utils.py
39
40
41
+- label: Entrypoints Integration (API Server 2)
42
+ timeout_in_minutes: 130
43
+ working_dir: "/vllm-workspace/tests"
44
+ source_file_dependencies:
45
+ - vllm/
46
+ - tests/tool_use
47
+ - tests/entrypoints/sleep
48
+ - tests/entrypoints/instrumentator
49
+ - tests/entrypoints/rpc
50
+ commands:
51
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
52
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
53
+ - pytest -v -s entrypoints/instrumentator
54
+ - pytest -v -s entrypoints/sleep
55
+ - pytest -v -s tool_use
56
+
57
- label: Entrypoints Integration (Pooling)
58
timeout_in_minutes: 50
59
working_dir: "/vllm-workspace/tests"
.buildkite/test_areas/tool_use.yaml
deleted
-13
@@ -1,13 +0,0 @@
1
-group: Tool use
2
-depends_on:
3
- - image-build
4
-steps:
5
-- label: OpenAI-Compatible Tool Use
6
- timeout_in_minutes: 35
7
- mirror_hardwares: [amdexperimental]
8
- fast_check: false
9
- source_file_dependencies:
10
- - vllm/
11
- - tests/tool_use
12
- commands:
13
- - pytest -v -s tool_use
tests/entrypoints/instrumentator/__init__.py
tests/entrypoints/instrumentator/test_metrics.py
renamed
+2
-3
@@ -14,11 +14,10 @@ import requests
14
from prometheus_client.parser import text_string_to_metric_families
15
from transformers import AutoTokenizer
16
17
+from tests.conftest import LocalAssetServer
18
+from tests.utils import RemoteOpenAIServer
19
from vllm import version
20
19
-from ...conftest import LocalAssetServer
20
-from ...utils import RemoteOpenAIServer
21
-
21
MODELS = {
22
"text": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
23
"multimodal": "HuggingFaceTB/SmolVLM-256M-Instruct",
tests/entrypoints/rpc/__init__.py
tests/entrypoints/rpc/test_collective_rpc.py
renamed
+1
-1
@@ -37,7 +37,7 @@ def server():
37
"--max-num-seqs",
38
"128",
39
"--worker-extension-cls",
40
- "tests.entrypoints.openai.test_collective_rpc.TestWorkerExtension",
40
+ "tests.entrypoints.rpc.test_collective_rpc.TestWorkerExtension",
41
]
42
with RemoteOpenAIServer(
43
MODEL_NAME,
tests/entrypoints/sleep/__init__.py
tests/entrypoints/sleep/test_sleep.py
renamed
+1
-1
@@ -4,7 +4,7 @@
4
import requests
5
from prometheus_client.parser import text_string_to_metric_families
6
7
-from ...utils import RemoteOpenAIServer
7
+from tests.utils import RemoteOpenAIServer
8
9
MODEL_NAME = "meta-llama/Llama-3.2-1B"
10
vllm/entrypoints/openai/api_server.py
+3
-95
@@ -17,21 +17,20 @@ from argparse import Namespace
17
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable
18
from contextlib import asynccontextmanager
19
from http import HTTPStatus
20
-from typing import Annotated, Any, Literal
20
+from typing import Annotated, Any
21
22
import model_hosting_container_standards.sagemaker as sagemaker_standards
23
import pydantic
24
import uvloop
25
-from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Query, Request
25
+from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
26
from fastapi.exceptions import RequestValidationError
27
from fastapi.middleware.cors import CORSMiddleware
28
-from fastapi.responses import JSONResponse, Response, StreamingResponse
28
+from fastapi.responses import JSONResponse, StreamingResponse
29
from starlette.concurrency import iterate_in_threadpool
30
from starlette.datastructures import URL, Headers, MutableHeaders, State
31
from starlette.types import ASGIApp, Message, Receive, Scope, Send
32
33
import vllm.envs as envs
34
-from vllm.config import VllmConfig
34
from vllm.engine.arg_utils import AsyncEngineArgs
35
from vllm.engine.protocol import EngineClient
36
from vllm.entrypoints.anthropic.protocol import (
@@ -639,97 +638,6 @@ async def create_translations(
638
return StreamingResponse(content=generator, media_type="text/event-stream")
639
640
642
-if envs.VLLM_SERVER_DEV_MODE:
643
- logger.warning(
644
- "SECURITY WARNING: Development endpoints are enabled! "
645
- "This should NOT be used in production!"
646
- )
647
-
648
- PydanticVllmConfig = pydantic.TypeAdapter(VllmConfig)
649
-
650
- @router.get("/server_info")
651
- async def show_server_info(
652
- raw_request: Request,
653
- config_format: Annotated[Literal["text", "json"], Query()] = "text",
654
- ):
655
- vllm_config: VllmConfig = raw_request.app.state.vllm_config
656
- server_info = {
657
- "vllm_config": str(vllm_config)
658
- if config_format == "text"
659
- else PydanticVllmConfig.dump_python(vllm_config, mode="json", fallback=str)
660
- # fallback=str is needed to handle e.g. torch.dtype
661
- }
662
- return JSONResponse(content=server_info)
663
-
664
- @router.post("/reset_prefix_cache")
665
- async def reset_prefix_cache(
666
- raw_request: Request,
667
- reset_running_requests: bool = Query(default=False),
668
- reset_external: bool = Query(default=False),
669
- ):
670
- """
671
- Reset the local prefix cache.
672
-
673
- Optionally, if the query parameter `reset_external=true`
674
- also resets the external (connector-managed) prefix cache.
675
-
676
- Note that we currently do not check if the prefix cache
677
- is successfully reset in the API server.
678
-
679
- Example:
680
- POST /reset_prefix_cache?reset_external=true
681
- """
682
- logger.info("Resetting prefix cache...")
683
-
684
- await engine_client(raw_request).reset_prefix_cache(
685
- reset_running_requests, reset_external
686
- )
687
- return Response(status_code=200)
688
-
689
- @router.post("/reset_mm_cache")
690
- async def reset_mm_cache(raw_request: Request):
691
- """
692
- Reset the multi-modal cache. Note that we currently do not check if the
693
- multi-modal cache is successfully reset in the API server.
694
- """
695
- logger.info("Resetting multi-modal cache...")
696
- await engine_client(raw_request).reset_mm_cache()
697
- return Response(status_code=200)
698
-
699
- @router.post("/collective_rpc")
700
- async def collective_rpc(raw_request: Request):
701
- try:
702
- body = await raw_request.json()
703
- except json.JSONDecodeError as e:
704
- raise HTTPException(
705
- status_code=HTTPStatus.BAD_REQUEST.value,
706
- detail=f"JSON decode error: {e}",
707
- ) from e
708
- method = body.get("method")
709
- if method is None:
710
- raise HTTPException(
711
- status_code=HTTPStatus.BAD_REQUEST.value,
712
- detail="Missing 'method' in request body",
713
- )
714
- # For security reason, only serialized string args/kwargs are passed.
715
- # User-defined `method` is responsible for deserialization if needed.
716
- args: list[str] = body.get("args", [])
717
- kwargs: dict[str, str] = body.get("kwargs", {})
718
- timeout: float | None = body.get("timeout")
719
- results = await engine_client(raw_request).collective_rpc(
720
- method=method, timeout=timeout, args=tuple(args), kwargs=kwargs
721
- )
722
- if results is None:
723
- return Response(status_code=200)
724
- response: list[Any] = []
725
- for result in results:
726
- if result is None or isinstance(result, dict | list):
727
- response.append(result)
728
- else:
729
- response.append(str(result))
730
- return JSONResponse(content={"results": response})
731
-
732
-
641
def load_log_config(log_config_file: str | None) -> dict | None:
642
if not log_config_file:
643
return None
vllm/entrypoints/serve/__init__.py
+29
@@ -4,8 +4,19 @@
4
5
from fastapi import FastAPI
6
7
+import vllm.envs as envs
8
+from vllm.logger import init_logger
9
+
10
+logger = init_logger(__name__)
11
+
12
13
def register_vllm_serve_api_routers(app: FastAPI):
14
+ if envs.VLLM_SERVER_DEV_MODE:
15
+ logger.warning(
16
+ "SECURITY WARNING: Development endpoints are enabled! "
17
+ "This should NOT be used in production!"
18
+ )
19
+
20
from vllm.entrypoints.serve.lora.api_router import (
21
attach_router as attach_lora_router,
22
)
@@ -29,6 +40,18 @@ def register_vllm_serve_api_routers(app: FastAPI):
40
41
attach_sleep_router(app)
42
43
+ from vllm.entrypoints.serve.rpc.api_router import (
44
+ attach_router as attach_rpc_router,
45
+ )
46
+
47
+ attach_rpc_router(app)
48
+
49
+ from vllm.entrypoints.serve.cache.api_router import (
50
+ attach_router as attach_cache_router,
51
+ )
52
+
53
+ attach_cache_router(app)
54
+
55
from vllm.entrypoints.serve.tokenize.api_router import (
56
attach_router as attach_tokenize_router,
57
)
@@ -58,3 +81,9 @@ def register_vllm_serve_api_routers(app: FastAPI):
81
)
82
83
attach_health_router(app)
84
+
85
+ from vllm.entrypoints.serve.instrumentator.server_info import (
86
+ attach_router as attach_server_info_router,
87
+ )
88
+
89
+ attach_server_info_router(app)
vllm/entrypoints/serve/cache/__init__.py
vllm/entrypoints/serve/cache/api_router.py
new
+61
@@ -0,0 +1,61 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from fastapi import APIRouter, FastAPI, Query, Request
6
+from fastapi.responses import Response
7
+
8
+import vllm.envs as envs
9
+from vllm.engine.protocol import EngineClient
10
+from vllm.logger import init_logger
11
+
12
+logger = init_logger(__name__)
13
+
14
+router = APIRouter()
15
+
16
+
17
+def engine_client(request: Request) -> EngineClient:
18
+ return request.app.state.engine_client
19
+
20
+
21
+@router.post("/reset_prefix_cache")
22
+async def reset_prefix_cache(
23
+ raw_request: Request,
24
+ reset_running_requests: bool = Query(default=False),
25
+ reset_external: bool = Query(default=False),
26
+):
27
+ """
28
+ Reset the local prefix cache.
29
+
30
+ Optionally, if the query parameter `reset_external=true`
31
+ also resets the external (connector-managed) prefix cache.
32
+
33
+ Note that we currently do not check if the prefix cache
34
+ is successfully reset in the API server.
35
+
36
+ Example:
37
+ POST /reset_prefix_cache?reset_external=true
38
+ """
39
+ logger.info("Resetting prefix cache...")
40
+
41
+ await engine_client(raw_request).reset_prefix_cache(
42
+ reset_running_requests, reset_external
43
+ )
44
+ return Response(status_code=200)
45
+
46
+
47
+@router.post("/reset_mm_cache")
48
+async def reset_mm_cache(raw_request: Request):
49
+ """
50
+ Reset the multi-modal cache. Note that we currently do not check if the
51
+ multi-modal cache is successfully reset in the API server.
52
+ """
53
+ logger.info("Resetting multi-modal cache...")
54
+ await engine_client(raw_request).reset_mm_cache()
55
+ return Response(status_code=200)
56
+
57
+
58
+def attach_router(app: FastAPI):
59
+ if not envs.VLLM_SERVER_DEV_MODE:
60
+ return
61
+ app.include_router(router)
vllm/entrypoints/serve/instrumentator/server_info.py
new
+40
@@ -0,0 +1,40 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from typing import Annotated, Literal
6
+
7
+import pydantic
8
+from fastapi import APIRouter, FastAPI, Query, Request
9
+from fastapi.responses import JSONResponse
10
+
11
+import vllm.envs as envs
12
+from vllm.config import VllmConfig
13
+from vllm.logger import init_logger
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+router = APIRouter()
19
+PydanticVllmConfig = pydantic.TypeAdapter(VllmConfig)
20
+
21
+
22
+@router.get("/server_info")
23
+async def show_server_info(
24
+ raw_request: Request,
25
+ config_format: Annotated[Literal["text", "json"], Query()] = "text",
26
+):
27
+ vllm_config: VllmConfig = raw_request.app.state.vllm_config
28
+ server_info = {
29
+ "vllm_config": str(vllm_config)
30
+ if config_format == "text"
31
+ else PydanticVllmConfig.dump_python(vllm_config, mode="json", fallback=str)
32
+ # fallback=str is needed to handle e.g. torch.dtype
33
+ }
34
+ return JSONResponse(content=server_info)
35
+
36
+
37
+def attach_router(app: FastAPI):
38
+ if not envs.VLLM_SERVER_DEV_MODE:
39
+ return
40
+ app.include_router(router)
vllm/entrypoints/serve/rpc/__init__.py
vllm/entrypoints/serve/rpc/api_router.py
new
+61
@@ -0,0 +1,61 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+from http import HTTPStatus
6
+from typing import Any
7
+
8
+from fastapi import APIRouter, FastAPI, HTTPException, Request
9
+from fastapi.responses import JSONResponse, Response
10
+
11
+import vllm.envs as envs
12
+from vllm.engine.protocol import EngineClient
13
+from vllm.logger import init_logger
14
+
15
+logger = init_logger(__name__)
16
+
17
+router = APIRouter()
18
+
19
+
20
+def engine_client(request: Request) -> EngineClient:
21
+ return request.app.state.engine_client
22
+
23
+
24
+@router.post("/collective_rpc")
25
+async def collective_rpc(raw_request: Request):
26
+ try:
27
+ body = await raw_request.json()
28
+ except json.JSONDecodeError as e:
29
+ raise HTTPException(
30
+ status_code=HTTPStatus.BAD_REQUEST.value,
31
+ detail=f"JSON decode error: {e}",
32
+ ) from e
33
+ method = body.get("method")
34
+ if method is None:
35
+ raise HTTPException(
36
+ status_code=HTTPStatus.BAD_REQUEST.value,
37
+ detail="Missing 'method' in request body",
38
+ )
39
+ # For security reason, only serialized string args/kwargs are passed.
40
+ # User-defined `method` is responsible for deserialization if needed.
41
+ args: list[str] = body.get("args", [])
42
+ kwargs: dict[str, str] = body.get("kwargs", {})
43
+ timeout: float | None = body.get("timeout")
44
+ results = await engine_client(raw_request).collective_rpc(
45
+ method=method, timeout=timeout, args=tuple(args), kwargs=kwargs
46
+ )
47
+ if results is None:
48
+ return Response(status_code=200)
49
+ response: list[Any] = []
50
+ for result in results:
51
+ if result is None or isinstance(result, dict | list):
52
+ response.append(result)
53
+ else:
54
+ response.append(str(result))
55
+ return JSONResponse(content={"results": response})
56
+
57
+
58
+def attach_router(app: FastAPI):
59
+ if not envs.VLLM_SERVER_DEV_MODE:
60
+ return
61
+ app.include_router(router)
vllm/entrypoints/serve/sleep/api_router.py
-4
@@ -52,9 +52,5 @@ async def is_sleeping(raw_request: Request):
52
def attach_router(app: FastAPI):
53
if not envs.VLLM_SERVER_DEV_MODE:
54
return
55
- logger.warning(
56
- "SECURITY WARNING: Development endpoints are enabled! "
57
- "This should NOT be used in production!"
58
- )
55
56
app.include_router(router)