@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48cf1e413c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

fix: deque mutated during iteration in abort_seq_group (#2371)

陈序 committed Jan 13, 2024 at 00:44 UTC 48cf1e413c42b29909077afe21c7b9e57996a1cf
1 file changed +16 -13
vllm/core/scheduler.py
+16 -13
@@ -104,21 +104,24 @@ class Scheduler:
104 request_id = (request_id, )
105 request_ids = set(request_id)
106 for state_queue in [self.waiting, self.running, self.swapped]:
107 - # We need to reverse the list as we are removing elements
108 - # from it as we iterate over it. If we don't do it,
109 - # indices will get messed up and we will skip over elements.
110 - for seq_group in reversed(state_queue):
107 + aborted_groups = []
108 + for seq_group in state_queue:
109 + if not request_ids:
110 + # Using 'break' here may add two extra iterations,
111 + # but is acceptable to reduce complexity .
112 + break
113 if seq_group.request_id in request_ids:
112 - # Remove the sequence group from the state queue.
113 - state_queue.remove(seq_group)
114 - for seq in seq_group.get_seqs():
115 - if seq.is_finished():
116 - continue
117 - seq.status = SequenceStatus.FINISHED_ABORTED
118 - self.free_seq(seq)
114 + # Appending aborted group into pending list.
115 + aborted_groups.append(seq_group)
116 request_ids.remove(seq_group.request_id)
120 - if not request_ids:
121 - return
117 + for aborted_group in aborted_groups:
118 + # Remove the sequence group from the state queue.
119 + state_queue.remove(aborted_group)
120 + for seq in seq_group.get_seqs():
121 + if seq.is_finished():
122 + continue
123 + seq.status = SequenceStatus.FINISHED_ABORTED
124 + self.free_seq(seq)
125
126 def has_unfinished_seqs(self) -> bool:
127 return self.waiting or self.running or self.swapped