A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Fix disagg hang caused by the prefill and decode communication issues (#12723)
Signed-off-by: Lu Fang <lufang@fb.com>
Lu Fang committed
Feb 7, 2025 at 16:39 UTC
45cbc4991dcf405c959f774d07e66e7e9ac71f0c
1 file changed
+40
-47
vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py
+40
-47
@@ -10,7 +10,6 @@
10
stop the prefill instance when the decode instance is slow.
11
"""
12
import threading
13
-import time
13
from collections import deque
14
from typing import Deque, List, Optional, Union
15
@@ -29,13 +28,13 @@ class SimpleBuffer(KVLookupBufferBase):
28
def __init__(self, signal_pipe: KVPipeBase, data_pipe: KVPipeBase,
29
buffer_size_thresh: float):
30
"""
32
- signal_pipe: on CPU
33
-
34
- NOTE: on-device recv will block all threads in the process, making the
35
- KV cache producer unable to listen to new request while transmitting
36
- KV cache. Luckily CPU recv only blocks the current thread so we use
31
+ signal_pipe: on CPU
32
+
33
+ NOTE: on-device recv will block all threads in the process, making the
34
+ KV cache producer unable to listen to new request while transmitting
35
+ KV cache. Luckily CPU recv only blocks the current thread so we use
36
CPU recv to listen to new request.
38
-
37
+
38
data_pipe: on device (e.g. GPU)
39
"""
40
@@ -43,7 +42,7 @@ class SimpleBuffer(KVLookupBufferBase):
42
43
self.buffer_size = 0
44
self.buffer_size_threshold = buffer_size_thresh
46
- self.buffer_lock = threading.Lock()
45
+ self.buffer_cv = threading.Condition()
46
self.signal_pipe = signal_pipe
47
self.data_pipe = data_pipe
48
self.request_handling_thread: Optional[threading.Thread] = None
@@ -116,11 +115,19 @@ class SimpleBuffer(KVLookupBufferBase):
115
hidden = hidden.clone()
116
117
buffer_item = [input_tokens, roi, key, value, hidden]
118
+ data_size = sum([self._get_element_size(data) for data in buffer_item])
119
+
120
+ with self.buffer_cv:
121
+ if self.buffer_size + data_size > self.buffer_size_threshold:
122
+ # log outside the while loop to avoid this message being logged
123
+ # repeatedly.
124
+ logger.debug("KV transfer buffer is full. Handling...")
125
+ while self.buffer_size + data_size > self.buffer_size_threshold:
126
+ self.buffer_cv.wait()
127
120
- with self.buffer_lock:
121
- for data in buffer_item:
122
- self.buffer_size += self._get_element_size(data)
128
+ self.buffer_size += data_size
129
self.buffer.append(buffer_item)
130
+ self.buffer_cv.notify()
131
132
def _is_end_signal(self, signal):
133
return signal is None
@@ -143,35 +150,31 @@ class SimpleBuffer(KVLookupBufferBase):
150
roi = (roi > 0.5)
151
tokens_roi_recver = [input_tokens, roi]
152
146
- matched_length = 0
147
-
148
- # perform input tokens and roi matching
149
- # FIXME: this matching is O(n), ideally it should be O(1)
150
- # but this buffer size won't (and shouldn't) be too large so
151
- # the fix is not urgent.
152
- with self.buffer_lock:
153
-
153
+ def is_buffer_available(
154
+ tokens_roi_recver: List[torch.Tensor], ) -> bool:
155
+ # perform input tokens and roi matching
156
+ # FIXME: this matching is O(n), ideally it should be O(1)
157
+ # but this buffer size won't (and shouldn't) be too large so
158
+ # the fix is not urgent.
159
for _ in range(len(self.buffer)):
155
-
156
- temp_length = self._matches(self.buffer[0],
157
- tokens_roi_recver)
158
- if temp_length > 0:
159
- matched_length = temp_length
160
- break
160
+ if self._matches(self.buffer[0],
161
+ tokens_roi_recver) > 0:
162
+ return True
163
# rotate the element we just accessed to the end
164
self.buffer.rotate(-1)
163
-
164
- if matched_length > 0:
165
- # need to clone the tensor
166
- # in case the tensor is freed before sending finishes
167
- matched_item = self.buffer.popleft()
168
- for tensor in matched_item:
169
- self._send_tensor_and_dec_size(tensor)
170
-
171
- else:
172
- # no match, just send None
173
- for _ in range(5):
174
- self.data_pipe.send_tensor(None)
165
+ return False
166
+
167
+ with self.buffer_cv:
168
+ while not is_buffer_available(tokens_roi_recver):
169
+ logger.debug(
170
+ "KV transfer buffer is not available. Waiting...")
171
+ self.buffer_cv.wait()
172
+ # need to clone the tensor
173
+ # in case the tensor is freed before sending finishes
174
+ matched_item = self.buffer.popleft()
175
+ for tensor in matched_item:
176
+ self._send_tensor_and_dec_size(tensor)
177
+ self.buffer_cv.notify()
178
179
except RuntimeError as e:
180
if 'Connection closed by peer' not in str(e):
@@ -208,20 +211,10 @@ class SimpleBuffer(KVLookupBufferBase):
211
212
return [input_tokens, roi, key, value, hidden]
213
211
- def full_handler(self):
212
- time.sleep(0.001)
213
-
214
def insert(self, input_tokens: torch.Tensor, roi: torch.Tensor,
215
key: torch.Tensor, value: torch.Tensor,
216
hidden: torch.Tensor) -> None:
217
218
- if self.buffer_size > self.buffer_size_threshold:
219
- # log outside the while loop to avoid this message being logged
220
- # repeatedly.
221
- logger.debug("KV transfer buffer is full. Handling...")
222
- while self.buffer_size > self.buffer_size_threshold:
223
- self.full_handler()
224
-
218
self._add_to_buffer(input_tokens, roi, key, value, hidden)
219
220
# when calling the insert, the current process is a sender