A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[core][misc] improve free_finished_seq_groups (#6865)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
youkaichao committed
Jul 30, 2024 at 14:32 UTC
6ca8031e7130f810ce7248286754a60044f65c73
1 file changed
+12
-7
vllm/core/scheduler.py
+12
-7
@@ -313,6 +313,7 @@ class Scheduler:
313
# Sequence groups finished requests ids since last step iteration.
314
# It lets the model know that any state associated with these requests
315
# can and must be released after the current step.
316
+ # This is used to evict the finished requests from the Mamba cache.
317
self._finished_requests_ids: List[str] = list()
318
# Time at previous scheduling step
319
self.prev_time = 0.0
@@ -374,6 +375,7 @@ class Scheduler:
375
for aborted_group in aborted_groups:
376
# Remove the sequence group from the state queue.
377
state_queue.remove(aborted_group)
378
+ # Remove the aborted request from the Mamba cache.
379
self._finished_requests_ids.append(aborted_group.request_id)
380
for seq in aborted_group.get_seqs():
381
if seq.is_finished():
@@ -1057,13 +1059,16 @@ class Scheduler:
1059
self.block_manager.free(seq)
1060
1061
def free_finished_seq_groups(self) -> None:
1060
- for queue in [self.running, self.swapped, self.waiting]:
1061
- self._finished_requests_ids += [
1062
- seq_group.request_id for seq_group in queue
1063
- if seq_group.is_finished()
1064
- ]
1065
- self.running = deque(seq_group for seq_group in self.running
1066
- if not seq_group.is_finished())
1062
+ remaining: Deque[SequenceGroup] = deque()
1063
+ for seq_group in self.running:
1064
+ if seq_group.is_finished():
1065
+ # Add the finished requests to the finished requests list.
1066
+ # This list will be used to update the Mamba cache in the
1067
+ # next step.
1068
+ self._finished_requests_ids.append(seq_group.request_id)
1069
+ else:
1070
+ remaining.append(seq_group)
1071
+ self.running = remaining
1072
1073
def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
1074
self.block_manager.allocate(seq_group)