A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix get_max_num_running_seqs for waiting and swapped seq groups (#1068)
Zhuohan Li committed
Sep 18, 2023 at 11:49 UTC
f029ef94d72528aaccd7d48f973031e9dff23447
1 file changed
+13
-2
vllm/sequence.py
+13
-2
@@ -250,8 +250,8 @@ class SequenceGroup:
250
# generation stage, we will have `best_of` sequences running.
251
return self.sampling_params.best_of
252
# At sampling stages, return the number of actual sequences
253
- # running.
254
- return self.num_seqs(status=SequenceStatus.RUNNING)
253
+ # that are not finished yet.
254
+ return self.num_unfinished_seqs()
255
256
def get_seqs(
257
self,
@@ -264,12 +264,23 @@ class SequenceGroup:
264
seq for seq in self.seqs_dict.values() if seq.status == status
265
]
266
267
+ def get_unfinished_seqs(self) -> List[Sequence]:
268
+ return [
269
+ seq for seq in self.seqs_dict.values() if not seq.is_finished()
270
+ ]
271
+
272
def get_finished_seqs(self) -> List[Sequence]:
273
return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
274
275
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
276
return len(self.get_seqs(status))
277
278
+ def num_unfinished_seqs(self) -> int:
279
+ return len(self.get_unfinished_seqs())
280
+
281
+ def num_finished_seqs(self) -> int:
282
+ return len(self.get_finished_seqs())
283
+
284
def find(self, seq_id: int) -> Sequence:
285
if seq_id not in self.seqs_dict:
286
raise ValueError(f"Sequence {seq_id} not found.")