@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d75c40734a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Fix] Keep `scheduler.running` as deque (#2523)

Nick Hill committed Jan 20, 2024 at 22:36 UTC d75c40734a96a10b30c7b2652d49f2a70030855b
1 file changed +2 -4
vllm/core/scheduler.py
+2 -4
@@ -332,10 +332,8 @@ class Scheduler:
332 self.block_manager.free(seq)
333
334 def free_finished_seq_groups(self) -> None:
335 - self.running = [
336 - seq_group for seq_group in self.running
337 - if not seq_group.is_finished()
338 - ]
335 + self.running = deque(seq_group for seq_group in self.running
336 + if not seq_group.is_finished())
337
338 def _allocate(self, seq_group: SequenceGroup) -> None:
339 self.block_manager.allocate(seq_group)