@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b1644f764f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix can_swap_in

Woosuk Kwon committed Feb 14, 2023 at 01:57 UTC b1644f764f8111e34872fa871745c97ee08a3226
1 file changed +2 -2
cacheflow/master/block_manager.py
+2 -2
@@ -147,12 +147,12 @@ class BlockSpaceManager:
147
148 def can_swap_in(self, seq_group: SequenceGroup) -> bool:
149 blocks = self._get_physical_blocks(seq_group)
150 - num_running_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
150 + num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
151 num_free_blocks = self.gpu_allocator.get_num_free_blocks()
152 # NOTE: Conservatively, we assume that every sequence will allocate
153 # at least one free block right after the swap-in.
154 # NOTE: This should match the logic in can_append().
155 - return len(blocks) + num_running_seqs <= num_free_blocks
155 + return len(blocks) + num_swapped_seqs <= num_free_blocks
156
157 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
158 # src_block_number -> dst_block_number