@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc7b8d1eef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Performance] e2e overheads reduction: Small followup diff (#7364)

Alexander Matveev committed Aug 9, 2024 at 11:49 UTC fc7b8d1eefcbe837a56b7c080509417fe5167e6c
2 files changed +5 -2
vllm/core/block_manager_v1.py
+2 -2
@@ -336,9 +336,9 @@ class BlockSpaceManagerV1(BlockSpaceManager):
336
337 # Assign the self-attention block tables for each sequence.
338 if len(wait_seqs) == 1:
339 - self.block_tables[wait_seqs[0].seq_id] = block_table
339 + self.block_tables[seq.seq_id] = block_table
340 else:
341 - for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
341 + for seq in wait_seqs:
342 self.block_tables[seq.seq_id] = block_table.copy()
343
344 # Allocate encoder sequence
vllm/sequence.py
+3
@@ -655,6 +655,9 @@ class SequenceGroup:
655 return [seq for seq in self.seqs if not seq.is_finished()]
656
657 def get_finished_seqs(self) -> List[Sequence]:
658 + if self.is_single_seq:
659 + return self.seqs if self.seqs[0].is_finished() else []
660 +
661 return [seq for seq in self.seqs if seq.is_finished()]
662
663 def update_num_computed_tokens(self, num_new_computed_tokens: int):