A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Fix raw_request extraction in load_aware_call decorator (#15382)
Signed-off-by: Daniel Salib <danielsalib@meta.com>
daniel-salib committed
Mar 25, 2025 at 22:29 UTC
5aefd6ac3169b7b56023549cfa9614274d6e15f0
1 file changed
+10
-3
vllm/entrypoints/utils.py
+10
-3
@@ -68,13 +68,20 @@ def decrement_server_load(request: Request):
68
def load_aware_call(func):
69
70
@functools.wraps(func)
71
- async def wrapper(*args, raw_request: Request, **kwargs):
71
+ async def wrapper(*args, **kwargs):
72
+ raw_request = kwargs.get("raw_request",
73
+ args[1] if len(args) > 1 else None)
74
+
75
+ if raw_request is None:
76
+ raise ValueError(
77
+ "raw_request required when server load tracking is enabled")
78
+
79
if not raw_request.app.state.enable_server_load_tracking:
73
- return await func(*args, raw_request=raw_request, **kwargs)
80
+ return await func(*args, **kwargs)
81
82
raw_request.app.state.server_load_metrics += 1
83
try:
77
- response = await func(*args, raw_request=raw_request, **kwargs)
84
+ response = await func(*args, **kwargs)
85
except Exception:
86
raw_request.app.state.server_load_metrics -= 1
87
raise