A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 06:00 UTC
[Frontend] Consolidate dev entrypoints. (#44170)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jun 2, 2026 at 21:30 UTC
b623f7ea95118c353eaed0301e9a6e7774b0e861
22 files changed
+90
-79
.buildkite/test-amd.yaml
+11
-13
@@ -1238,14 +1238,11 @@ steps:
1238
working_dir: "/vllm-workspace/tests"
1239
source_file_dependencies:
1240
- vllm/
1241
- - tests/entrypoints/rpc
1242
- - tests/entrypoints/serve/instrumentator
1243
- - tests/tool_use
1241
+ - tests/entrypoints/serve
1242
commands:
1243
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1246
- - pytest -v -s entrypoints/serve/instrumentator
1247
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
1248
- - pytest -v -s tool_use
1244
+ - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
1245
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
1246
1247
- label: Entrypoints Integration (API Server openai - Part 1) # TBD
1248
timeout_in_minutes: 180
@@ -1276,11 +1273,13 @@ steps:
1273
- tests/entrypoints/openai
1274
- tests/entrypoints/test_chat_utils
1275
- tests/entrypoints/generate
1276
+ - tests/tool_use
1277
commands:
1278
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1279
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
1280
- pytest -v -s entrypoints/test_chat_utils.py
1281
- pytest -v -s entrypoints/generate
1282
+ - pytest -v -s tool_use
1283
1284
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
1285
timeout_in_minutes: 180
@@ -1370,7 +1369,7 @@ steps:
1369
- vllm/platforms/rocm.py
1370
commands:
1371
- pytest -v -s entrypoints/openai/tool_parsers
1373
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
1372
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/offline_mode --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
1373
1374
- label: OpenAI API correctness # TBD
1375
timeout_in_minutes: 180
@@ -2747,14 +2746,11 @@ steps:
2746
working_dir: "/vllm-workspace/tests"
2747
source_file_dependencies:
2748
- vllm/
2750
- - tests/entrypoints/rpc
2751
- - tests/entrypoints/serve/instrumentator
2752
- - tests/tool_use
2749
+ - tests/entrypoints/serve
2750
commands:
2751
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2755
- - pytest -v -s entrypoints/serve/instrumentator
2756
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
2757
- - pytest -v -s tool_use
2752
+ - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
2753
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
2754
2755
- label: Entrypoints Integration (API Server openai - Part 1) # TBD
2756
timeout_in_minutes: 180
@@ -2785,11 +2781,13 @@ steps:
2781
- tests/entrypoints/openai
2782
- tests/entrypoints/test_chat_utils
2783
- tests/entrypoints/generate
2784
+ - tests/tool_use
2785
commands:
2786
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2787
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
2788
- pytest -v -s entrypoints/test_chat_utils.py
2789
- pytest -v -s entrypoints/generate
2790
+ - pytest -v -s tool_use
2791
2792
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
2793
timeout_in_minutes: 180
.buildkite/test_areas/entrypoints.yaml
+6
-8
@@ -11,7 +11,7 @@ steps:
11
- tests/entrypoints/
12
commands:
13
- pytest -v -s entrypoints/openai/tool_parsers
14
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
14
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/offline_mode --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
15
16
- label: Entrypoints Integration (LLM)
17
key: entrypoints-integration-llm
@@ -61,10 +61,12 @@ steps:
61
- tests/entrypoints/openai
62
- tests/entrypoints/test_chat_utils
63
- tests/entrypoints/generate
64
+ - tests/tool_use
65
commands:
66
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
67
- pytest -v -s entrypoints/test_chat_utils.py
68
- pytest -v -s entrypoints/generate
69
+ - pytest -v -s tool_use
70
mirror:
71
amd:
72
device: mi325_1
@@ -100,14 +102,11 @@ steps:
102
working_dir: "/vllm-workspace/tests"
103
source_file_dependencies:
104
- vllm/
103
- - tests/entrypoints/rpc
104
- - tests/entrypoints/serve/instrumentator
105
- - tests/tool_use
105
+ - tests/entrypoints/serve
106
commands:
107
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
108
- - pytest -v -s entrypoints/serve/instrumentator
109
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
110
- - pytest -v -s tool_use
108
+ - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
109
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
110
mirror:
111
amd:
112
device: mi325_1
@@ -155,6 +154,5 @@ steps:
154
source_file_dependencies:
155
- csrc/
156
- vllm/entrypoints/openai/
158
- - vllm/model_executor/models/whisper.py
157
commands: # LMEval
158
- pytest -s entrypoints/openai/correctness/
.buildkite/test_areas/rust_frontend.yaml
+4
-4
@@ -45,19 +45,19 @@ steps:
45
- vllm/entrypoints/serve/
46
- vllm/v1/engine/
47
- tests/utils.py
48
- # - tests/entrypoints/rpc/test_collective_rpc.py
48
+ # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
49
- tests/entrypoints/serve/disagg/test_serving_tokens.py
50
- tests/entrypoints/serve/instrumentator/test_basic.py
51
- tests/entrypoints/serve/instrumentator/test_metrics.py
52
- # - tests/entrypoints/serve/instrumentator/test_sleep.py
52
+ # - tests/entrypoints/serve/dev/test_sleep.py
53
commands:
54
- export VLLM_USE_RUST_FRONTEND=1
55
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
56
- # - pytest -v -s entrypoints/rpc/test_collective_rpc.py
56
+ # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
57
- pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
58
- pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
59
- pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
60
- # - pytest -v -s entrypoints/serve/instrumentator/test_sleep.py
60
+ # - pytest -v -s entrypoints/serve/dev/test_sleep.py
61
62
- label: Rust Frontend Core Correctness
63
timeout_in_minutes: 30
docs/serving/online_serving/README.md
+32
-2
@@ -100,14 +100,44 @@ For further details on renderer APIs, please refer to [this page](renderer.md).
100
- `/version` - Version information
101
- `/load` - Server load metrics
102
103
-## Sleep Mode APIs
103
+## Server in development mode
104
+
105
+When using the flag VLLM_SERVER_DEV_MODE=1, you enable development endpoints.
106
+
107
+**SECURITY WARNING: These endpoints should NOT be used in production!**
108
+
109
+### Cache Management APIs
110
+
111
+- `/reset_prefix_cache` - Reset prefix cache (can disrupt service)
112
+- `/reset_mm_cache` - Reset multimodal cache (can disrupt service)
113
+- `/reset_encoder_cache` - Reset encoder cache (can disrupt service)
114
+
115
+### Weight Transfer APIs (RL Training)
116
+
117
+For further details on Weight Transfer, please refer to [this page](../../training/weight_transfer/README.md).
118
+
119
+- `/pause` - Pause generation (causes denial of service)
120
+- `/resume` - Resume generation
121
+- `/is_paused` - Check if generation is paused
122
+- `/init_weight_transfer_engine` - Initialize weight transfer engine for RLHF
123
+- `/update_weights` - Update model weights (can alter model behavior)
124
+- `/get_world_size` - Get distributed world size
125
+
126
+### Collective RPC
127
+
128
+- `/collective_rpc` - Execute arbitrary RPC methods on the engine (extremely dangerous)
129
+
130
+### Server info
131
+
132
+- `/server_info` - Get detailed server configuration
133
+
134
+### Sleep Mode APIs
135
136
For further details on sleep mode, please refer to [this page](../../features/sleep_mode.md).
137
138
- `/sleep` - Put engine to sleep (causes denial of service)
139
- `/wake_up` - Wake engine from sleep
140
- `/is_sleeping` - Check if engine is sleeping
110
-- `/collective_rpc` - Execute arbitrary RPC methods on the engine (extremely dangerous)
141
142
## Chat Template
143
tests/entrypoints/serve/dev/__init__.py
renamed
tests/entrypoints/serve/dev/rpc/__init__.py
renamed
tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
renamed
+1
-1
@@ -37,7 +37,7 @@ def server():
37
"--max-num-seqs",
38
"128",
39
"--worker-extension-cls",
40
- "tests.entrypoints.rpc.test_collective_rpc.TestWorkerExtension",
40
+ "tests.entrypoints.serve.dev.rpc.test_collective_rpc.TestWorkerExtension",
41
]
42
with RemoteOpenAIServer(
43
MODEL_NAME,
tests/entrypoints/serve/dev/test_sleep.py
renamed
vllm/entrypoints/openai/api_server.py
+6
-9
@@ -43,9 +43,7 @@ from vllm.entrypoints.openai.server_utils import (
43
validation_exception_handler,
44
)
45
from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
46
-from vllm.entrypoints.serve.elastic_ep.middleware import (
47
- ScalingMiddleware,
48
-)
46
+from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
47
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
48
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
49
from vllm.entrypoints.utils import (
@@ -195,6 +193,11 @@ def build_app(
193
194
register_sagemaker_api_router(app, supported_tasks, model_config)
195
196
+ if envs.VLLM_SERVER_DEV_MODE:
197
+ from vllm.entrypoints.serve import register_vllm_dev_api_routers
198
+
199
+ register_vllm_dev_api_routers(app)
200
+
201
if "generate" in supported_tasks:
202
from vllm.entrypoints.generate.api_router import (
203
register_generate_api_routers,
@@ -208,12 +211,6 @@ def build_app(
211
212
attach_disagg_router(app)
213
211
- from vllm.entrypoints.serve.rlhf.api_router import (
212
- attach_router as attach_rlhf_router,
213
- )
214
-
215
- attach_rlhf_router(app)
216
-
214
from vllm.entrypoints.serve.elastic_ep.api_router import (
215
attach_router as elastic_ep_attach_router,
216
)
vllm/entrypoints/serve/__init__.py
+25
-21
@@ -3,18 +3,15 @@
3
4
from fastapi import FastAPI
5
6
-import vllm.envs as envs
6
from vllm.logger import init_logger
7
8
logger = init_logger(__name__)
9
10
11
def register_vllm_serve_api_routers(app: FastAPI):
13
- if envs.VLLM_SERVER_DEV_MODE:
14
- logger.warning(
15
- "SECURITY WARNING: Development endpoints are enabled! "
16
- "This should NOT be used in production!"
17
- )
12
+ from .instrumentator import register_instrumentator_api_routers
13
+
14
+ register_instrumentator_api_routers(app)
15
16
from vllm.entrypoints.serve.lora.api_router import (
17
attach_router as attach_lora_router,
@@ -28,30 +25,37 @@ def register_vllm_serve_api_routers(app: FastAPI):
25
26
attach_profile_router(app)
27
31
- from vllm.entrypoints.serve.sleep.api_router import (
32
- attach_router as attach_sleep_router,
28
+ from vllm.entrypoints.serve.tokenize.api_router import (
29
+ attach_router as attach_tokenize_router,
30
)
31
35
- attach_sleep_router(app)
32
+ attach_tokenize_router(app)
33
37
- from vllm.entrypoints.serve.rpc.api_router import (
38
- attach_router as attach_rpc_router,
39
- )
34
41
- attach_rpc_router(app)
42
-
43
- from vllm.entrypoints.serve.cache.api_router import (
44
- attach_router as attach_cache_router,
35
+def register_vllm_dev_api_routers(app: FastAPI):
36
+ logger.warning(
37
+ "SECURITY WARNING: Development endpoints are enabled! "
38
+ "This should NOT be used in production!"
39
)
40
41
+ from .dev.cache.api_router import attach_router as attach_cache_router
42
+
43
attach_cache_router(app)
44
49
- from vllm.entrypoints.serve.tokenize.api_router import (
50
- attach_router as attach_tokenize_router,
45
+ from .dev.rlhf.api_router import attach_router as attach_rlhf_router
46
+
47
+ attach_rlhf_router(app)
48
+
49
+ from .dev.rpc.api_router import attach_router as attach_rpc_router
50
+
51
+ attach_rpc_router(app)
52
+
53
+ from .dev.server_info.api_router import (
54
+ attach_router as attach_server_info_router,
55
)
56
53
- attach_tokenize_router(app)
57
+ attach_server_info_router(app)
58
55
- from .instrumentator import register_instrumentator_api_routers
59
+ from .dev.sleep.api_router import attach_router as attach_sleep_router
60
57
- register_instrumentator_api_routers(app)
61
+ attach_sleep_router(app)
vllm/entrypoints/serve/dev/__init__.py
renamed
vllm/entrypoints/serve/dev/cache/__init__.py
renamed
vllm/entrypoints/serve/dev/cache/api_router.py
renamed
-3
@@ -5,7 +5,6 @@
5
from fastapi import APIRouter, FastAPI, Query, Request
6
from fastapi.responses import Response
7
8
-import vllm.envs as envs
8
from vllm.engine.protocol import EngineClient
9
from vllm.logger import init_logger
10
@@ -67,6 +66,4 @@ async def reset_encoder_cache(raw_request: Request):
66
67
68
def attach_router(app: FastAPI):
70
- if not envs.VLLM_SERVER_DEV_MODE:
71
- return
69
app.include_router(router)
vllm/entrypoints/serve/dev/rlhf/__init__.py
renamed
vllm/entrypoints/serve/dev/rlhf/api_router.py
renamed
-3
@@ -8,7 +8,6 @@ from typing import Annotated
8
from fastapi import APIRouter, FastAPI, HTTPException, Query, Request
9
from fastapi.responses import JSONResponse
10
11
-import vllm.envs as envs
11
from vllm.distributed.weight_transfer.base import (
12
WeightTransferInitRequest,
13
WeightTransferUpdateRequest,
@@ -186,6 +185,4 @@ async def get_world_size(
185
186
187
def attach_router(app: FastAPI):
189
- if not envs.VLLM_SERVER_DEV_MODE:
190
- return
188
app.include_router(router)
vllm/entrypoints/serve/dev/rpc/__init__.py
vllm/entrypoints/serve/dev/rpc/api_router.py
renamed
-3
@@ -8,7 +8,6 @@ from typing import Any
8
from fastapi import APIRouter, FastAPI, HTTPException, Request
9
from fastapi.responses import JSONResponse, Response
10
11
-import vllm.envs as envs
11
from vllm.engine.protocol import EngineClient
12
from vllm.logger import init_logger
13
@@ -56,6 +55,4 @@ async def collective_rpc(raw_request: Request):
55
56
57
def attach_router(app: FastAPI):
59
- if not envs.VLLM_SERVER_DEV_MODE:
60
- return
58
app.include_router(router)
vllm/entrypoints/serve/dev/server_info/__init__.py
vllm/entrypoints/serve/dev/server_info/api_router.py
renamed
+5
-1
@@ -7,7 +7,7 @@ import functools
7
from typing import Annotated, Literal
8
9
import pydantic
10
-from fastapi import APIRouter, Query, Request
10
+from fastapi import APIRouter, FastAPI, Query, Request
11
from fastapi.responses import JSONResponse
12
13
import vllm.envs as envs
@@ -57,3 +57,7 @@ async def show_server_info(
57
"system_env": await asyncio.to_thread(_get_system_env_info_cached),
58
}
59
return JSONResponse(content=server_info)
60
+
61
+
62
+def attach_router(app: FastAPI):
63
+ app.include_router(router)
vllm/entrypoints/serve/dev/sleep/__init__.py
vllm/entrypoints/serve/dev/sleep/api_router.py
renamed
-4
@@ -5,7 +5,6 @@
5
from fastapi import APIRouter, FastAPI, Request
6
from fastapi.responses import JSONResponse, Response
7
8
-import vllm.envs as envs
8
from vllm.engine.protocol import EngineClient
9
from vllm.logger import init_logger
10
@@ -50,7 +49,4 @@ async def is_sleeping(raw_request: Request):
49
50
51
def attach_router(app: FastAPI):
53
- if not envs.VLLM_SERVER_DEV_MODE:
54
- return
55
-
52
app.include_router(router)
vllm/entrypoints/serve/instrumentator/__init__.py
-7
@@ -3,8 +3,6 @@
3
4
from fastapi import FastAPI
5
6
-from vllm import envs
7
-
6
7
def register_instrumentator_api_routers(app: FastAPI):
8
from .basic import router as basic_router
@@ -22,8 +20,3 @@ def register_instrumentator_api_routers(app: FastAPI):
20
from .offline_docs import attach_router as offline_docs_attach_router
21
22
offline_docs_attach_router(app)
25
-
26
- if envs.VLLM_SERVER_DEV_MODE:
27
- from .server_info import router as server_info_router
28
-
29
- app.include_router(server_info_router)