@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 526df28fb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[BugFix] Fix a bug in counting running sequences (#266)

Woosuk Kwon committed Jun 26, 2023 at 13:09 UTC 526df28fb250c90970cd3db0f68a21797dbe555f
1 file changed +6 -2
vllm/core/scheduler.py
+6 -2
@@ -156,7 +156,9 @@ class Scheduler:
156 # The total number of sequences in the RUNNING state should not
157 # exceed the maximum number of sequences.
158 num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
159 - num_curr_seqs = len(self.running)
159 + num_curr_seqs = sum(
160 + seq_group.num_seqs(status=SequenceStatus.RUNNING)
161 + for seq_group in self.running)
162 if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
163 break
164
@@ -198,7 +200,9 @@ class Scheduler:
200 # The total number of sequences in the RUNNING state should not
201 # exceed the maximum number of sequences.
202 num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
201 - num_curr_seqs = len(self.running)
203 + num_curr_seqs = sum(
204 + seq_group.num_seqs(status=SequenceStatus.RUNNING)
205 + for seq_group in self.running)
206 if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
207 break
208