@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d2a68364c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BugFix] Fix abort_seq_group (#2463)

ljss committed Jan 19, 2024 at 07:10 UTC d2a68364c473a3167a1c2b90f947bb611322a867
1 file changed +2 -2
vllm/core/scheduler.py
+2 -2
@@ -108,7 +108,7 @@ class Scheduler:
108 request_id = (request_id, )
109 request_ids = set(request_id)
110 for state_queue in [self.waiting, self.running, self.swapped]:
111 - aborted_groups = []
111 + aborted_groups: List[SequenceGroup] = []
112 for seq_group in state_queue:
113 if not request_ids:
114 # Using 'break' here may add two extra iterations,
@@ -121,7 +121,7 @@ class Scheduler:
121 for aborted_group in aborted_groups:
122 # Remove the sequence group from the state queue.
123 state_queue.remove(aborted_group)
124 - for seq in seq_group.get_seqs():
124 + for seq in aborted_group.get_seqs():
125 if seq.is_finished():
126 continue
127 seq.status = SequenceStatus.FINISHED_ABORTED