A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add cache watermark to avoid frequent cache eviction (#11)
Woosuk Kwon committed
Mar 29, 2023 at 16:38 UTC
64e0e383148a613c327d4bf9e866b7a185df8277
1 file changed
+8
-2
cacheflow/master/block_manager.py
+8
-2
@@ -60,11 +60,15 @@ class BlockSpaceManager:
60
block_size: int,
61
num_gpu_blocks: int,
62
num_cpu_blocks: int,
63
+ watermark: float = 0.01,
64
) -> None:
65
self.block_size = block_size
66
self.num_total_gpu_blocks = num_gpu_blocks
67
self.num_total_cpu_blocks = num_cpu_blocks
68
+ self.watermark = watermark
69
+ assert watermark >= 0.0
70
71
+ self.watermark_blocks = int(watermark * num_gpu_blocks)
72
self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
73
self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
74
@@ -76,7 +80,8 @@ class BlockSpaceManager:
80
seq = seq_group.seqs[0]
81
num_required_blocks = len(seq.logical_token_blocks)
82
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
79
- return num_required_blocks <= num_free_gpu_blocks
83
+ # Use watermark to avoid frequent cache eviction.
84
+ return num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks
85
86
def allocate(self, seq_group: SequenceGroup) -> None:
87
# NOTE: Here we assume that all sequences in the group have the same prompt.
@@ -154,7 +159,8 @@ class BlockSpaceManager:
159
# NOTE: Conservatively, we assume that every sequence will allocate
160
# at least one free block right after the swap-in.
161
# NOTE: This should match the logic in can_append().
157
- return len(blocks) + num_swapped_seqs <= num_free_blocks
162
+ num_required_blocks = len(blocks) + num_swapped_seqs
163
+ return num_free_blocks - num_required_blocks >= self.watermark_blocks
164
165
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
166
# CPU block -> GPU block.