A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 15:15 UTC
[Frontend][CI] Consolidate instrumentator entrypoints (#34123)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Feb 10, 2026 at 15:30 UTC
dab1de9f3895a153a7bc2ce7ef7782ba7818a146
16 files changed
+64
-74
.buildkite/test-amd.yaml
+4
-4
@@ -132,7 +132,7 @@ steps:
132
- tests/entrypoints/
133
commands:
134
- pytest -v -s entrypoints/openai/tool_parsers
135
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/instrumentator --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
135
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/rpc --ignore=entrypoints/instrumentator --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
136
137
- label: Entrypoints Integration Test (LLM) # 30min
138
timeout_in_minutes: 40
@@ -179,14 +179,14 @@ steps:
179
torch_nightly: true
180
source_file_dependencies:
181
- vllm/
182
- - tests/entrypoints/sleep
182
- tests/entrypoints/rpc
183
+ - tests/entrypoints/instrumentator
184
- tests/tool_use
185
commands:
186
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
187
- - pytest -v -s entrypoints/sleep
187
+ - pytest -v -s entrypoints/instrumentator
188
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
189
- pytest -v -s tool_use
189
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
190
191
- label: Entrypoints Integration Test (Pooling)
192
timeout_in_minutes: 50
.buildkite/test-pipeline.yaml
+5
-5
@@ -118,7 +118,7 @@ steps:
118
- tests/entrypoints/
119
commands:
120
- pytest -v -s entrypoints/openai/tool_parsers
121
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
121
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
122
123
- label: Entrypoints Integration Test (LLM) # 30min
124
timeout_in_minutes: 40
@@ -148,7 +148,7 @@ steps:
148
- tests/entrypoints/test_chat_utils
149
commands:
150
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
151
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
151
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/instrumentator --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
152
- pytest -v -s entrypoints/test_chat_utils.py
153
154
- label: Entrypoints Integration Test (API Server 2)
@@ -159,13 +159,13 @@ steps:
159
torch_nightly: true
160
source_file_dependencies:
161
- vllm/
162
- - tests/entrypoints/sleep
162
- tests/entrypoints/rpc
163
+ - tests/entrypoints/instrumentator
164
- tests/tool_use
165
commands:
166
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
167
- - pytest -v -s entrypoints/sleep
168
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
167
+ - pytest -v -s entrypoints/instrumentator
168
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
169
- pytest -v -s tool_use
170
171
- label: Entrypoints Integration Test (Pooling)
.buildkite/test_areas/entrypoints.yaml
+3
-5
@@ -42,15 +42,13 @@ steps:
42
working_dir: "/vllm-workspace/tests"
43
source_file_dependencies:
44
- vllm/
45
- - tests/tool_use
46
- - tests/entrypoints/sleep
47
- - tests/entrypoints/instrumentator
45
- tests/entrypoints/rpc
46
+ - tests/entrypoints/instrumentator
47
+ - tests/tool_use
48
commands:
49
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
51
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
50
- pytest -v -s entrypoints/instrumentator
53
- - pytest -v -s entrypoints/sleep
51
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
52
- pytest -v -s tool_use
53
54
- label: Entrypoints Integration (Pooling)
tests/entrypoints/instrumentator/test_basic.py
renamed
tests/entrypoints/instrumentator/test_optional_middleware.py
renamed
tests/entrypoints/instrumentator/test_orca_metrics.py
renamed
tests/entrypoints/instrumentator/test_sleep.py
renamed
tests/entrypoints/sleep/__init__.py
vllm/entrypoints/openai/api_server.py
+18
-4
@@ -178,10 +178,6 @@ def build_app(
178
app = FastAPI(lifespan=lifespan)
179
app.state.args = args
180
181
- from vllm.entrypoints.openai.basic.api_router import register_basic_api_routers
182
-
183
- register_basic_api_routers(app)
184
-
181
from vllm.entrypoints.serve import register_vllm_serve_api_routers
182
183
register_vllm_serve_api_routers(app)
@@ -205,6 +201,24 @@ def build_app(
201
202
register_generate_api_routers(app)
203
204
+ from vllm.entrypoints.serve.disagg.api_router import (
205
+ attach_router as attach_disagg_router,
206
+ )
207
+
208
+ attach_disagg_router(app)
209
+
210
+ from vllm.entrypoints.serve.rlhf.api_router import (
211
+ attach_router as attach_rlhf_router,
212
+ )
213
+
214
+ attach_rlhf_router(app)
215
+
216
+ from vllm.entrypoints.serve.elastic_ep.api_router import (
217
+ attach_router as elastic_ep_attach_router,
218
+ )
219
+
220
+ elastic_ep_attach_router(app)
221
+
222
if "transcription" in supported_tasks:
223
from vllm.entrypoints.openai.speech_to_text.api_router import (
224
attach_router as register_speech_to_text_api_router,
vllm/entrypoints/openai/basic/__init__.py
vllm/entrypoints/sagemaker/api_router.py
+1
-1
@@ -10,10 +10,10 @@ import pydantic
10
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
11
from fastapi.responses import JSONResponse, Response
12
13
-from vllm.entrypoints.openai.basic.api_router import base
13
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
14
from vllm.entrypoints.openai.engine.serving import OpenAIServing
15
from vllm.entrypoints.openai.utils import validate_json_request
16
+from vllm.entrypoints.serve.instrumentator.basic import base
17
from vllm.entrypoints.serve.instrumentator.health import health
18
from vllm.tasks import POOLING_TASKS, SupportedTask
19
vllm/entrypoints/serve/__init__.py
+2
-39
@@ -22,12 +22,6 @@ def register_vllm_serve_api_routers(app: FastAPI):
22
23
attach_lora_router(app)
24
25
- from vllm.entrypoints.serve.elastic_ep.api_router import (
26
- attach_router as attach_elastic_ep_router,
27
- )
28
-
29
- attach_elastic_ep_router(app)
30
-
25
from vllm.entrypoints.serve.profile.api_router import (
26
attach_router as attach_profile_router,
27
)
@@ -58,37 +52,6 @@ def register_vllm_serve_api_routers(app: FastAPI):
52
53
attach_tokenize_router(app)
54
61
- from vllm.entrypoints.serve.disagg.api_router import (
62
- attach_router as attach_disagg_router,
63
- )
64
-
65
- attach_disagg_router(app)
66
-
67
- from vllm.entrypoints.serve.rlhf.api_router import (
68
- attach_router as attach_rlhf_router,
69
- )
70
-
71
- attach_rlhf_router(app)
72
-
73
- from vllm.entrypoints.serve.instrumentator.metrics import (
74
- attach_router as attach_metrics_router,
75
- )
76
-
77
- attach_metrics_router(app)
78
-
79
- from vllm.entrypoints.serve.instrumentator.health import (
80
- attach_router as attach_health_router,
81
- )
82
-
83
- attach_health_router(app)
84
-
85
- from vllm.entrypoints.serve.instrumentator.offline_docs import (
86
- attach_router as attach_offline_docs_router,
87
- )
88
-
89
- attach_offline_docs_router(app)
90
- from vllm.entrypoints.serve.instrumentator.server_info import (
91
- attach_router as attach_server_info_router,
92
- )
55
+ from .instrumentator import register_instrumentator_api_routers
56
94
- attach_server_info_router(app)
57
+ register_instrumentator_api_routers(app)
vllm/entrypoints/serve/instrumentator/__init__.py
+29
@@ -0,0 +1,29 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from fastapi import FastAPI
5
+
6
+from vllm import envs
7
+
8
+
9
+def register_instrumentator_api_routers(app: FastAPI):
10
+ from .basic import router as basic_router
11
+
12
+ app.include_router(basic_router)
13
+
14
+ from .health import router as health_router
15
+
16
+ app.include_router(health_router)
17
+
18
+ from .metrics import attach_router as metrics_attach_router
19
+
20
+ metrics_attach_router(app)
21
+
22
+ from .offline_docs import attach_router as offline_docs_attach_router
23
+
24
+ offline_docs_attach_router(app)
25
+
26
+ if envs.VLLM_SERVER_DEV_MODE:
27
+ from .server_info import router as server_info_router
28
+
29
+ app.include_router(server_info_router)
vllm/entrypoints/serve/instrumentator/basic.py
renamed
+1
-5
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from fastapi import APIRouter, FastAPI, Request
4
+from fastapi import APIRouter, Request
5
from fastapi.responses import JSONResponse
6
7
from vllm.engine.protocol import EngineClient
@@ -55,7 +55,3 @@ async def get_server_load_metrics(request: Request):
55
async def show_version():
56
ver = {"version": VLLM_VERSION}
57
return JSONResponse(content=ver)
58
-
59
-
60
-def register_basic_api_routers(app: FastAPI):
61
- app.include_router(router)
vllm/entrypoints/serve/instrumentator/health.py
-4
@@ -27,7 +27,3 @@ async def health(raw_request: Request) -> Response:
27
return Response(status_code=200)
28
except EngineDeadError:
29
return Response(status_code=503)
30
-
31
-
32
-def attach_router(app):
33
- app.include_router(router)
vllm/entrypoints/serve/instrumentator/server_info.py
+1
-7
@@ -7,7 +7,7 @@ import functools
7
from typing import Annotated, Literal
8
9
import pydantic
10
-from fastapi import APIRouter, FastAPI, Query, Request
10
+from fastapi import APIRouter, Query, Request
11
from fastapi.responses import JSONResponse
12
13
import vllm.envs as envs
@@ -57,9 +57,3 @@ async def show_server_info(
57
"system_env": await asyncio.to_thread(_get_system_env_info_cached),
58
}
59
return JSONResponse(content=server_info)
60
-
61
-
62
-def attach_router(app: FastAPI):
63
- if not envs.VLLM_SERVER_DEV_MODE:
64
- return
65
- app.include_router(router)