A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix can_swap_in
Woosuk Kwon committed
Feb 14, 2023 at 01:57 UTC
b1644f764f8111e34872fa871745c97ee08a3226
1 file changed
+2
-2
cacheflow/master/block_manager.py
+2
-2
@@ -147,12 +147,12 @@ class BlockSpaceManager:
147
148
def can_swap_in(self, seq_group: SequenceGroup) -> bool:
149
blocks = self._get_physical_blocks(seq_group)
150
- num_running_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
150
+ num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
151
num_free_blocks = self.gpu_allocator.get_num_free_blocks()
152
# NOTE: Conservatively, we assume that every sequence will allocate
153
# at least one free block right after the swap-in.
154
# NOTE: This should match the logic in can_append().
155
- return len(blocks) + num_running_seqs <= num_free_blocks
155
+ return len(blocks) + num_swapped_seqs <= num_free_blocks
156
157
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
158
# src_block_number -> dst_block_number