A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Fix] Keep `scheduler.running` as deque (#2523)
Nick Hill committed
Jan 20, 2024 at 22:36 UTC
d75c40734a96a10b30c7b2652d49f2a70030855b
1 file changed
+2
-4
vllm/core/scheduler.py
+2
-4
@@ -332,10 +332,8 @@ class Scheduler:
332
self.block_manager.free(seq)
333
334
def free_finished_seq_groups(self) -> None:
335
- self.running = [
336
- seq_group for seq_group in self.running
337
- if not seq_group.is_finished()
338
- ]
335
+ self.running = deque(seq_group for seq_group in self.running
336
+ if not seq_group.is_finished())
337
338
def _allocate(self, seq_group: SequenceGroup) -> None:
339
self.block_manager.allocate(seq_group)