A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[BugFix] Fix abort_seq_group (#2463)
ljss committed
Jan 19, 2024 at 07:10 UTC
d2a68364c473a3167a1c2b90f947bb611322a867
1 file changed
+2
-2
vllm/core/scheduler.py
+2
-2
@@ -108,7 +108,7 @@ class Scheduler:
108
request_id = (request_id, )
109
request_ids = set(request_id)
110
for state_queue in [self.waiting, self.running, self.swapped]:
111
- aborted_groups = []
111
+ aborted_groups: List[SequenceGroup] = []
112
for seq_group in state_queue:
113
if not request_ids:
114
# Using 'break' here may add two extra iterations,
@@ -121,7 +121,7 @@ class Scheduler:
121
for aborted_group in aborted_groups:
122
# Remove the sequence group from the state queue.
123
state_queue.remove(aborted_group)
124
- for seq in seq_group.get_seqs():
124
+ for seq in aborted_group.get_seqs():
125
if seq.is_finished():
126
continue
127
seq.status = SequenceStatus.FINISHED_ABORTED