@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f029ef94d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix get_max_num_running_seqs for waiting and swapped seq groups (#1068)

Zhuohan Li committed Sep 18, 2023 at 11:49 UTC f029ef94d72528aaccd7d48f973031e9dff23447
1 file changed +13 -2
vllm/sequence.py
+13 -2
@@ -250,8 +250,8 @@ class SequenceGroup:
250 # generation stage, we will have `best_of` sequences running.
251 return self.sampling_params.best_of
252 # At sampling stages, return the number of actual sequences
253 - # running.
254 - return self.num_seqs(status=SequenceStatus.RUNNING)
253 + # that are not finished yet.
254 + return self.num_unfinished_seqs()
255
256 def get_seqs(
257 self,
@@ -264,12 +264,23 @@ class SequenceGroup:
264 seq for seq in self.seqs_dict.values() if seq.status == status
265 ]
266
267 + def get_unfinished_seqs(self) -> List[Sequence]:
268 + return [
269 + seq for seq in self.seqs_dict.values() if not seq.is_finished()
270 + ]
271 +
272 def get_finished_seqs(self) -> List[Sequence]:
273 return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
274
275 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
276 return len(self.get_seqs(status))
277
278 + def num_unfinished_seqs(self) -> int:
279 + return len(self.get_unfinished_seqs())
280 +
281 + def num_finished_seqs(self) -> int:
282 + return len(self.get_finished_seqs())
283 +
284 def find(self, seq_id: int) -> Sequence:
285 if seq_id not in self.seqs_dict:
286 raise ValueError(f"Sequence {seq_id} not found.")