A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[P/D] Avoid stranding blocks in P when aborted in D's waiting queue (#19223)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jun 25, 2025 at 15:19 UTC
55c65ab495f5d270f65f89dcc737e9694b278002
1 file changed
+14
-1
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+14
-1
@@ -298,8 +298,21 @@ class NixlConnectorScheduler:
298
logger.debug(
299
"NIXLConnector request_finished, request_status=%s, "
300
"kv_transfer_params=%s", request.status, params)
301
+ if not params:
302
+ return False, None
303
+
304
+ if params.get("do_remote_prefill"):
305
+ # If do_remote_prefill is still True when the request is finished,
306
+ # update_state_after_alloc must not have been called (the request
307
+ # must have been aborted before it was scheduled).
308
+ # To avoid stranding the prefill blocks in the prefill instance,
309
+ # we must add empty block_ids to _reqs_need_recv so that our
310
+ # worker side will notify and free blocks in the prefill instance.
311
+ self._reqs_need_recv[request.request_id] = (request, [])
312
+ params["do_remote_prefill"] = False
313
+ return False, None
314
302
- if (params is None or not params.get("do_remote_decode")
315
+ if (not params.get("do_remote_decode")
316
or request.status != RequestStatus.FINISHED_LENGTH_CAPPED):
317
return False, None
318