A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
decrement server_load on listen for disconnect (#18784)
Signed-off-by: Daniel Salib <danielsalib@meta.com>
daniel-salib committed
May 28, 2025 at 07:15 UTC
6e4cea1cc56da6d4a558a50196c5698c36385890
1 file changed
+5
vllm/entrypoints/utils.py
+5
@@ -26,6 +26,11 @@ async def listen_for_disconnect(request: Request) -> None:
26
while True:
27
message = await request.receive()
28
if message["type"] == "http.disconnect":
29
+ if request.app.state.enable_server_load_tracking:
30
+ # on timeout/cancellation the BackgroundTask in load_aware_call
31
+ # cannot decrement the server load metrics.
32
+ # Must be decremented by with_cancellation instead.
33
+ request.app.state.server_load_metrics -= 1
34
break
35
36