A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[BugFix] Fix ordering of KVConnector finished send/rcv sets (#18211)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
May 16, 2025 at 09:20 UTC
0ceaebf87b3c21a70432556d777e1c83919bdf6f
2 files changed
+5
-4
vllm/distributed/kv_transfer/kv_connector/v1/base.py
+2
-1
@@ -183,7 +183,8 @@ class KVConnectorBase_V1(ABC):
183
finished generating tokens.
184
185
Returns:
186
- ids of requests that have finished asynchronous (recving, sending).
186
+ ids of requests that have finished asynchronous transfer,
187
+ tuple of (sending/saving ids, recving/loading ids).
188
The finished saves/sends req ids must belong to a set provided in a
189
call to this method (this call or a prior one).
190
"""
vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
+3
-3
@@ -103,10 +103,10 @@ class MultiConnector(KVConnectorBase_V1):
103
def get_finished(
104
self, finished_req_ids: set[str]
105
) -> tuple[Optional[set[str]], Optional[set[str]]]:
106
- finished_recving: set[str] = set()
106
finished_sending: set[str] = set()
107
+ finished_recving: set[str] = set()
108
for c in self._connectors:
109
- recving, sending = c.get_finished(finished_req_ids)
109
+ sending, recving = c.get_finished(finished_req_ids)
110
if not recving and not sending:
111
continue
112
# Aggregate finished recving request ids.
@@ -125,7 +125,7 @@ class MultiConnector(KVConnectorBase_V1):
125
else:
126
self._extra_async_saves[req_id] = extra_pending - 1
127
128
- return finished_recving or None, finished_sending or None
128
+ return finished_sending or None, finished_recving or None
129
130
# ==============================
131
# Scheduler-side methods