@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ca8031e71
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[core][misc] improve free_finished_seq_groups (#6865)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

youkaichao committed Jul 30, 2024 at 14:32 UTC 6ca8031e7130f810ce7248286754a60044f65c73
1 file changed +12 -7
vllm/core/scheduler.py
+12 -7
@@ -313,6 +313,7 @@ class Scheduler:
313 # Sequence groups finished requests ids since last step iteration.
314 # It lets the model know that any state associated with these requests
315 # can and must be released after the current step.
316 + # This is used to evict the finished requests from the Mamba cache.
317 self._finished_requests_ids: List[str] = list()
318 # Time at previous scheduling step
319 self.prev_time = 0.0
@@ -374,6 +375,7 @@ class Scheduler:
375 for aborted_group in aborted_groups:
376 # Remove the sequence group from the state queue.
377 state_queue.remove(aborted_group)
378 + # Remove the aborted request from the Mamba cache.
379 self._finished_requests_ids.append(aborted_group.request_id)
380 for seq in aborted_group.get_seqs():
381 if seq.is_finished():
@@ -1057,13 +1059,16 @@ class Scheduler:
1059 self.block_manager.free(seq)
1060
1061 def free_finished_seq_groups(self) -> None:
1060 - for queue in [self.running, self.swapped, self.waiting]:
1061 - self._finished_requests_ids += [
1062 - seq_group.request_id for seq_group in queue
1063 - if seq_group.is_finished()
1064 - ]
1065 - self.running = deque(seq_group for seq_group in self.running
1066 - if not seq_group.is_finished())
1062 + remaining: Deque[SequenceGroup] = deque()
1063 + for seq_group in self.running:
1064 + if seq_group.is_finished():
1065 + # Add the finished requests to the finished requests list.
1066 + # This list will be used to update the Mamba cache in the
1067 + # next step.
1068 + self._finished_requests_ids.append(seq_group.request_id)
1069 + else:
1070 + remaining.append(seq_group)
1071 + self.running = remaining
1072
1073 def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
1074 self.block_manager.allocate(seq_group)