@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64e0e38314
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add cache watermark to avoid frequent cache eviction (#11)

Woosuk Kwon committed Mar 29, 2023 at 16:38 UTC 64e0e383148a613c327d4bf9e866b7a185df8277
1 file changed +8 -2
cacheflow/master/block_manager.py
+8 -2
@@ -60,11 +60,15 @@ class BlockSpaceManager:
60 block_size: int,
61 num_gpu_blocks: int,
62 num_cpu_blocks: int,
63 + watermark: float = 0.01,
64 ) -> None:
65 self.block_size = block_size
66 self.num_total_gpu_blocks = num_gpu_blocks
67 self.num_total_cpu_blocks = num_cpu_blocks
68 + self.watermark = watermark
69 + assert watermark >= 0.0
70
71 + self.watermark_blocks = int(watermark * num_gpu_blocks)
72 self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
73 self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
74
@@ -76,7 +80,8 @@ class BlockSpaceManager:
80 seq = seq_group.seqs[0]
81 num_required_blocks = len(seq.logical_token_blocks)
82 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
79 - return num_required_blocks <= num_free_gpu_blocks
83 + # Use watermark to avoid frequent cache eviction.
84 + return num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks
85
86 def allocate(self, seq_group: SequenceGroup) -> None:
87 # NOTE: Here we assume that all sequences in the group have the same prompt.
@@ -154,7 +159,8 @@ class BlockSpaceManager:
159 # NOTE: Conservatively, we assume that every sequence will allocate
160 # at least one free block right after the swap-in.
161 # NOTE: This should match the logic in can_append().
157 - return len(blocks) + num_swapped_seqs <= num_free_blocks
162 + num_required_blocks = len(blocks) + num_swapped_seqs
163 + return num_free_blocks - num_required_blocks >= self.watermark_blocks
164
165 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
166 # CPU block -> GPU block.