A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[BugFix] Fix a bug in counting running sequences (#266)
Woosuk Kwon committed
Jun 26, 2023 at 13:09 UTC
526df28fb250c90970cd3db0f68a21797dbe555f
1 file changed
+6
-2
vllm/core/scheduler.py
+6
-2
@@ -156,7 +156,9 @@ class Scheduler:
156
# The total number of sequences in the RUNNING state should not
157
# exceed the maximum number of sequences.
158
num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
159
- num_curr_seqs = len(self.running)
159
+ num_curr_seqs = sum(
160
+ seq_group.num_seqs(status=SequenceStatus.RUNNING)
161
+ for seq_group in self.running)
162
if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
163
break
164
@@ -198,7 +200,9 @@ class Scheduler:
200
# The total number of sequences in the RUNNING state should not
201
# exceed the maximum number of sequences.
202
num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
201
- num_curr_seqs = len(self.running)
203
+ num_curr_seqs = sum(
204
+ seq_group.num_seqs(status=SequenceStatus.RUNNING)
205
+ for seq_group in self.running)
206
if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
207
break
208