A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Performance] e2e overheads reduction: Small followup diff (#7364)
Alexander Matveev committed
Aug 9, 2024 at 11:49 UTC
fc7b8d1eefcbe837a56b7c080509417fe5167e6c
2 files changed
+5
-2
vllm/core/block_manager_v1.py
+2
-2
@@ -336,9 +336,9 @@ class BlockSpaceManagerV1(BlockSpaceManager):
336
337
# Assign the self-attention block tables for each sequence.
338
if len(wait_seqs) == 1:
339
- self.block_tables[wait_seqs[0].seq_id] = block_table
339
+ self.block_tables[seq.seq_id] = block_table
340
else:
341
- for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
341
+ for seq in wait_seqs:
342
self.block_tables[seq.seq_id] = block_table.copy()
343
344
# Allocate encoder sequence
vllm/sequence.py
+3
@@ -655,6 +655,9 @@ class SequenceGroup:
655
return [seq for seq in self.seqs if not seq.is_finished()]
656
657
def get_finished_seqs(self) -> List[Sequence]:
658
+ if self.is_single_seq:
659
+ return self.seqs if self.seqs[0].is_finished() else []
660
+
661
return [seq for seq in self.seqs if seq.is_finished()]
662
663
def update_num_computed_tokens(self, num_new_computed_tokens: int):