A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Frontend] track responsesAPI server_load (#32323)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jan 14, 2026 at 20:00 UTC
00e6402d56fb258e6958381b1f3ceb34217ba830
2 files changed
+8
vllm/entrypoints/openai/api_server.py
+4
@@ -259,6 +259,10 @@ def engine_client(request: Request) -> EngineClient:
259
async def get_server_load_metrics(request: Request):
260
# This endpoint returns the current server load metrics.
261
# It tracks requests utilizing the GPU from the following routes:
262
+ # - /v1/responses
263
+ # - /v1/responses/{response_id}
264
+ # - /v1/responses/{response_id}/cancel
265
+ # - /v1/messages
266
# - /v1/chat/completions
267
# - /v1/completions
268
# - /v1/audio/transcriptions
vllm/entrypoints/openai/responses/api_router.py
+4
@@ -17,6 +17,7 @@ from vllm.entrypoints.openai.responses.protocol import (
17
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
18
from vllm.entrypoints.openai.utils import validate_json_request
19
from vllm.entrypoints.utils import (
20
+ load_aware_call,
21
with_cancellation,
22
)
23
from vllm.logger import init_logger
@@ -54,6 +55,7 @@ async def _convert_stream_to_sse_events(
55
},
56
)
57
@with_cancellation
58
+@load_aware_call
59
async def create_responses(request: ResponsesRequest, raw_request: Request):
60
handler = responses(raw_request)
61
if handler is None:
@@ -79,6 +81,7 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
81
82
83
@router.get("/v1/responses/{response_id}")
84
+@load_aware_call
85
async def retrieve_responses(
86
response_id: str,
87
raw_request: Request,
@@ -113,6 +116,7 @@ async def retrieve_responses(
116
117
118
@router.post("/v1/responses/{response_id}/cancel")
119
+@load_aware_call
120
async def cancel_responses(response_id: str, raw_request: Request):
121
handler = responses(raw_request)
122
if handler is None: