A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
fix: deque mutated during iteration in abort_seq_group (#2371)
陈序 committed
Jan 13, 2024 at 00:44 UTC
48cf1e413c42b29909077afe21c7b9e57996a1cf
1 file changed
+16
-13
vllm/core/scheduler.py
+16
-13
@@ -104,21 +104,24 @@ class Scheduler:
104
request_id = (request_id, )
105
request_ids = set(request_id)
106
for state_queue in [self.waiting, self.running, self.swapped]:
107
- # We need to reverse the list as we are removing elements
108
- # from it as we iterate over it. If we don't do it,
109
- # indices will get messed up and we will skip over elements.
110
- for seq_group in reversed(state_queue):
107
+ aborted_groups = []
108
+ for seq_group in state_queue:
109
+ if not request_ids:
110
+ # Using 'break' here may add two extra iterations,
111
+ # but is acceptable to reduce complexity .
112
+ break
113
if seq_group.request_id in request_ids:
112
- # Remove the sequence group from the state queue.
113
- state_queue.remove(seq_group)
114
- for seq in seq_group.get_seqs():
115
- if seq.is_finished():
116
- continue
117
- seq.status = SequenceStatus.FINISHED_ABORTED
118
- self.free_seq(seq)
114
+ # Appending aborted group into pending list.
115
+ aborted_groups.append(seq_group)
116
request_ids.remove(seq_group.request_id)
120
- if not request_ids:
121
- return
117
+ for aborted_group in aborted_groups:
118
+ # Remove the sequence group from the state queue.
119
+ state_queue.remove(aborted_group)
120
+ for seq in seq_group.get_seqs():
121
+ if seq.is_finished():
122
+ continue
123
+ seq.status = SequenceStatus.FINISHED_ABORTED
124
+ self.free_seq(seq)
125
126
def has_unfinished_seqs(self) -> bool:
127
return self.waiting or self.running or self.swapped