A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[P/D] Log warnings related to prefill KV expiry (#21753)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jul 28, 2025 at 19:40 UTC
7d44c691b0eb77640ae42b7842d799ec886efc75
1 file changed
+12
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+12
@@ -1025,6 +1025,11 @@ class NixlConnectorWorker:
1025
# Sorted dict, oldest requests are put first so we can exit early.
1026
if now < expires:
1027
break
1028
+ count = self.consumer_notification_counts_by_req.pop(req_id, 0)
1029
+ logger.warning(
1030
+ "Releasing expired KV blocks for request %s which were "
1031
+ "retrieved by %d decode worker(s) within %d seconds.", req_id,
1032
+ count, envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT)
1033
del self._reqs_to_send[req_id]
1034
done_sending.add(req_id)
1035
@@ -1040,6 +1045,13 @@ class NixlConnectorWorker:
1045
for notifs in self.nixl_wrapper.get_new_notifs().values():
1046
for notif in notifs:
1047
req_id, tp_ratio = notif.decode("utf-8").rsplit(":", 1)
1048
+ if req_id not in self._reqs_to_send:
1049
+ logger.error(
1050
+ "Potentially invalid KV blocks for "
1051
+ "unrecognized request %s were retrieved by "
1052
+ "a decode worker. They may have expired.", req_id)
1053
+ continue
1054
+
1055
self.consumer_notification_counts_by_req[req_id] += 1
1056
# Wait all consumers (D) to be done reading before freeing.
1057
if self.consumer_notification_counts_by_req[req_id] == int(