@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9474e89ba4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[PREFIX CACHING FOLLOW UP] A bunch of fixes to block allocator performance when automatic prefix caching is disabled (#3357)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

ElizaWszola committed Mar 20, 2024 at 08:11 UTC 9474e89ba4ecae253b585eb6b3e1d85f4e108f01
4 files changed +165 -121
tests/core/test_block_manager.py
+8 -6
@@ -4,7 +4,7 @@ from typing import List
4
5 from vllm import SamplingParams
6 from vllm.block import PhysicalTokenBlock
7 -from vllm.core.block_manager import (BlockAllocator, BlockSpaceManager,
7 +from vllm.core.block_manager import (UncachedBlockAllocator, BlockSpaceManager,
8 AllocStatus)
9 from vllm.utils import Device
10 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
@@ -15,7 +15,8 @@ from .utils import create_dummy_prompt
15 def test_block_allocator_allocate():
16 block_size = 4
17 num_cpu_blocks = 4
18 - cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
18 + cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
19 + num_cpu_blocks)
20
21 # Allocate all available cpu blocks.
22 num_free = num_cpu_blocks
@@ -24,7 +25,7 @@ def test_block_allocator_allocate():
25 block = cpu_allocator.allocate()
26 num_free -= 1
27
27 - assert block.block_hash not in cpu_allocator.evictor
28 + assert block not in cpu_allocator.free_blocks
29 assert cpu_allocator.get_num_free_blocks() == num_free
30
31 with pytest.raises(ValueError):
@@ -34,14 +35,15 @@ def test_block_allocator_allocate():
35 def test_block_allocator_free():
36 block_size = 4
37 num_cpu_blocks = 4
37 - cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
38 + cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
39 + num_cpu_blocks)
40
41 # Allocate all available cpu blocks.
42 blocks: List[PhysicalTokenBlock] = []
43 for _ in range(num_cpu_blocks):
44 block = cpu_allocator.allocate()
45 blocks.append(block)
44 - assert block.block_hash not in cpu_allocator.evictor
46 + assert block not in cpu_allocator.free_blocks
47
48 # Free all allocated cpu blocks.
49 num_free = 0
@@ -49,7 +51,7 @@ def test_block_allocator_free():
51 for block in blocks:
52 cpu_allocator.free(block)
53 num_free += 1
52 - assert block.block_hash in cpu_allocator.evictor
54 + assert block in cpu_allocator.free_blocks
55 assert cpu_allocator.get_num_free_blocks() == num_free
56
57 with pytest.raises(ValueError):
tests/prefix_caching/test_prefix_caching.py
+3 -9
@@ -4,7 +4,7 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 import pytest
6
7 -from vllm.core.block_manager import BlockAllocator
7 +from vllm.core.block_manager import CachedBlockAllocator
8 from vllm.utils import Device
9
10
@@ -15,10 +15,7 @@ def test_block_allocator(
15 num_blocks: int,
16 ):
17 block_hash = 1
18 - block_allocator = BlockAllocator(Device.CPU,
19 - block_size,
20 - num_blocks,
21 - enable_caching=True)
18 + block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
19
20 # Allocate two PysicalTokenBlocks with the same hash and check
21 # that they are the same PhysicalTokenBlock
@@ -45,10 +42,7 @@ def test_block_allocator(
42 @pytest.mark.parametrize("num_blocks", [16])
43 def test_eviction(num_blocks: int, ):
44 block_size = 16
48 - block_allocator = BlockAllocator(Device.CPU,
49 - block_size,
50 - num_blocks,
51 - enable_caching=True)
45 + block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
46 blocks = []
47
48 for i in range(num_blocks):
vllm/core/block_manager.py
+146 -43
@@ -3,6 +3,7 @@ import enum
3 from itertools import count, takewhile
4 from os.path import commonprefix
5 from typing import Dict, List, Optional, Set, Tuple
6 +from abc import ABC, abstractmethod
7
8 from vllm.block import BlockTable, PhysicalTokenBlock
9 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -10,7 +11,7 @@ from vllm.utils import Device
11 from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
12
13
13 -class BlockAllocator:
14 +class BlockAllocatorBase(ABC):
15 """Manages free physical token blocks for a device.
16
17 The allocator maintains a list of free blocks and allocates a block when
@@ -18,23 +19,57 @@ class BlockAllocator:
19 the reference count becomes zero, the block is added back to the free list.
20 """
21
22 + @abstractmethod
23 def __init__(self,
24 device: Device,
25 block_size: int,
26 num_blocks: int,
25 - eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
26 - enable_caching: bool = False) -> None:
27 + eviction_policy: EvictionPolicy = EvictionPolicy.LRU):
28 + pass
29 +
30 + @abstractmethod
31 + def allocate(self,
32 + block_hash: Optional[int] = None,
33 + num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
34 + pass
35 +
36 + @abstractmethod
37 + def free(self, block: PhysicalTokenBlock) -> None:
38 + pass
39 +
40 + @abstractmethod
41 + def get_num_free_blocks(self) -> int:
42 + pass
43 +
44 + @abstractmethod
45 + def contains_block(self, block_hash: int) -> bool:
46 + pass
47 +
48 + @abstractmethod
49 + def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
50 + pass
51 +
52 +
53 +class CachedBlockAllocator(BlockAllocatorBase):
54 + """Manages free physical token blocks for a device.
55 +
56 + The allocator maintains a list of free blocks and allocates a block when
57 + requested. When a block is freed, its reference count is decremented. If
58 + the reference count becomes zero, the block is added back to the free list.
59 + """
60 +
61 + def __init__(self,
62 + device: Device,
63 + block_size: int,
64 + num_blocks: int,
65 + eviction_policy: EvictionPolicy = EvictionPolicy.LRU) -> None:
66 self.device = device
67 self.block_size = block_size
68 self.num_blocks = num_blocks
30 - self.enable_caching = enable_caching
69
70 self.current_num_blocks = 0
71 self.cached_blocks: Dict[int, PhysicalTokenBlock] = {}
72
35 - # Switch over to FIFO eviction when caching is disabled
36 - if not self.enable_caching:
37 - eviction_policy = EvictionPolicy.FIFO
73 self.evictor: Evictor = make_evictor(eviction_policy)
74
75 self.default_hash_ctr = count()
@@ -57,13 +92,6 @@ class BlockAllocator:
92 def allocate(self,
93 block_hash: Optional[int] = None,
94 num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
60 - # If caching is disabled, just allocate a new block and return it
61 - if not self.enable_caching:
62 - block = self.allocate_block(next(self.default_hash_ctr),
63 - num_hashed_tokens)
64 - block.ref_count += 1
65 - return block
66 -
95 if block_hash is None:
96 block_hash = next(self.default_hash_ctr)
97 if block_hash in self.evictor:
@@ -90,9 +118,8 @@ class BlockAllocator:
118 assert block.block_hash not in self.evictor
119 self.evictor.add(block)
120
93 - # If caching is enabled, remove the block from the cached_blocks
94 - if self.enable_caching:
95 - del self.cached_blocks[block.block_hash]
121 + # Remove the block from the cached_blocks
122 + del self.cached_blocks[block.block_hash]
123
124 def get_num_free_blocks(self) -> int:
125 return (self.num_blocks - self.current_num_blocks +
@@ -102,14 +129,68 @@ class BlockAllocator:
129 return block_hash in self.cached_blocks or block_hash in self.evictor
130
131 def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
105 - # If caching is enabled, update the hash of block and the
106 - # cached_blocks dictionary.
107 - if self.enable_caching:
108 - assert not self.contains_block(block_hash)
109 - old_hash = block.block_hash
110 - block.block_hash = block_hash
111 - del self.cached_blocks[old_hash]
112 - self.cached_blocks[block_hash] = block
132 + # Update the hash of block and the cached_blocks dictionary.
133 + assert not self.contains_block(block_hash)
134 + old_hash = block.block_hash
135 + block.block_hash = block_hash
136 + del self.cached_blocks[old_hash]
137 + self.cached_blocks[block_hash] = block
138 +
139 +
140 +class UncachedBlockAllocator(BlockAllocatorBase):
141 + """Manages free physical token blocks for a device.
142 +
143 + The allocator maintains a list of free blocks and allocates a block when
144 + requested. When a block is freed, its reference count is decremented. If
145 + the reference count becomes zero, the block is added back to the free list.
146 + """
147 +
148 + def __init__(
149 + self,
150 + device: Device,
151 + block_size: int,
152 + num_blocks: int,
153 + ) -> None:
154 + self.device = device
155 + self.block_size = block_size
156 + self.num_blocks = num_blocks
157 +
158 + # Initialize the free blocks.
159 + self.free_blocks: BlockTable = []
160 + for i in range(num_blocks):
161 + block = PhysicalTokenBlock(device=device,
162 + block_number=i,
163 + block_size=block_size,
164 + block_hash=-1,
165 + num_hashed_tokens=0)
166 + self.free_blocks.append(block)
167 +
168 + def allocate(self,
169 + block_hash: Optional[int] = None,
170 + num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
171 + if not self.free_blocks:
172 + raise ValueError("Out of memory! No free blocks are available.")
173 + block = self.free_blocks.pop()
174 + block.ref_count = 1
175 + return block
176 +
177 + def free(self, block: PhysicalTokenBlock) -> None:
178 + if block.ref_count == 0:
179 + raise ValueError(f"Double free! {block} is already freed.")
180 + block.ref_count -= 1
181 + if block.ref_count == 0:
182 + self.free_blocks.append(block)
183 +
184 + def get_num_free_blocks(self) -> int:
185 + return len(self.free_blocks)
186 +
187 + def contains_block(self, block_hash: int) -> bool:
188 + raise NotImplementedError(
189 + "Invalid codepath for uncached block allocator.")
190 +
191 + def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
192 + raise NotImplementedError(
193 + "Invalid codepath for uncached block allocator.")
194
195
196 class AllocStatus(enum.Enum):
@@ -142,6 +223,10 @@ class BlockSpaceManager:
223 self.num_total_gpu_blocks = num_gpu_blocks
224 self.num_total_cpu_blocks = num_cpu_blocks
225
226 + if enable_caching and sliding_window is not None:
227 + raise NotImplementedError(
228 + "Sliding window is not allowed with prefix caching enabled!")
229 +
230 self.block_sliding_window = None
231 if sliding_window is not None:
232 assert sliding_window % block_size == 0, (sliding_window,
@@ -154,14 +239,17 @@ class BlockSpaceManager:
239 self.enable_caching = enable_caching
240
241 self.watermark_blocks = int(watermark * num_gpu_blocks)
157 - self.gpu_allocator = BlockAllocator(Device.GPU,
158 - block_size,
159 - num_gpu_blocks,
160 - enable_caching=enable_caching)
161 - self.cpu_allocator = BlockAllocator(Device.CPU,
162 - block_size,
163 - num_cpu_blocks,
164 - enable_caching=enable_caching)
242 +
243 + if self.enable_caching:
244 + self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
245 + num_gpu_blocks)
246 + self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
247 + num_cpu_blocks)
248 + else:
249 + self.gpu_allocator = UncachedBlockAllocator(
250 + Device.GPU, block_size, num_gpu_blocks)
251 + self.cpu_allocator = UncachedBlockAllocator(
252 + Device.CPU, block_size, num_cpu_blocks)
253 # Mapping: seq_id -> BlockTable.
254 self.block_tables: Dict[int, BlockTable] = {}
255
@@ -198,10 +286,16 @@ class BlockSpaceManager:
286 if (self.block_sliding_window is not None
287 and logical_idx >= self.block_sliding_window):
288 block = block_table[logical_idx % self.block_sliding_window]
201 - else:
289 + # Set the reference counts of the token blocks.
290 + block.ref_count = seq_group.num_seqs()
291 + elif self.enable_caching:
292 block = self.gpu_allocator.allocate(
293 seq.hash_of_block(logical_idx),
294 seq.num_hashed_tokens_of_block(logical_idx))
295 + else:
296 + block = self.gpu_allocator.allocate()
297 + # Set the reference counts of the token blocks.
298 + block.ref_count = seq_group.num_seqs()
299 block_table.append(block)
300
301 # Assign the block table for each sequence.
@@ -220,8 +314,10 @@ class BlockSpaceManager:
314 seq: Sequence,
315 last_block: PhysicalTokenBlock,
316 ) -> PhysicalTokenBlock:
223 - # Compute a new hash for the block so that it can be shared by
224 - # other Sequences
317 + assert self.enable_caching
318 +
319 + # Compute a new hash for the block so that it can be shared by other
320 + # Sequences
321 new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
322
323 # if new_hash is already in the cached table, then free last_block
@@ -254,6 +350,8 @@ class BlockSpaceManager:
350 self,
351 seq: Sequence,
352 ) -> PhysicalTokenBlock:
353 + if not self.enable_caching:
354 + return self.gpu_allocator.allocate()
355 block_hash: Optional[int] = None
356 if (self._is_last_block_full(seq)):
357 block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
@@ -293,10 +391,12 @@ class BlockSpaceManager:
391 assert last_block.device == Device.GPU
392 if last_block.ref_count == 1:
393 # Not shared with other sequences. Appendable.
296 - # If the last block is now complete, promote it to a full block so
297 - # that it can be shared
298 - new_block = self._maybe_promote_last_block(seq, last_block)
299 - block_table[-1] = new_block
394 + if self.enable_caching:
395 + # If the last block is now complete, we may reuse an old block
396 + # to save memory.
397 + maybe_new_block = self._maybe_promote_last_block(
398 + seq, last_block)
399 + block_table[-1] = maybe_new_block
400 return None
401 else:
402 # The last block is shared with other sequences.
@@ -440,9 +540,12 @@ class BlockSpaceManager:
540 seq: Sequence,
541 access_time: float,
542 ) -> None:
443 - block_table = self.block_tables[seq.seq_id]
444 - for block in block_table:
445 - block.last_accessed = access_time
543 + if self.enable_caching:
544 + # Update the last accessed time of all the blocks accessed
545 + # in this step.
546 + block_table = self.block_tables[seq.seq_id]
547 + for block in block_table:
548 + block.last_accessed = access_time
549
550 def compute_full_blocks_in_seq(self, seq: Sequence):
551 if seq.seq_id not in self.block_tables:
vllm/core/evictor.py
+8 -63
@@ -1,5 +1,5 @@
1 import enum
2 -from typing import Dict, List, Optional
2 +from typing import Dict
3 from abc import ABC, abstractmethod, abstractproperty
4
5 from vllm.block import PhysicalTokenBlock
@@ -10,7 +10,6 @@ class EvictionPolicy(enum.Enum):
10 Evictor subclass.
11 """
12 LRU = enum.auto()
13 - FIFO = enum.auto()
13
14
15 class Evictor(ABC):
@@ -66,37 +65,18 @@ class LRUEvictor(Evictor):
65
66 # TODO: The performance of this evict function can be optimized further.
67 def evict(self) -> PhysicalTokenBlock:
69 - free_blocks: List[PhysicalTokenBlock] = list(self.free_table.values())
70 - if len(free_blocks) == 0:
68 + if len(self.free_table) == 0:
69 raise ValueError("No usable cache memory left")
70 + free_blocks = self.free_table.values()
71
73 - # Find lowest timestamp
74 - lowest_timestamp = free_blocks[0].last_accessed
75 - for block in free_blocks:
76 - if block.last_accessed < lowest_timestamp:
77 - lowest_timestamp = block.last_accessed
72 + # Get evicted block
73 + evicted_block: PhysicalTokenBlock = next(iter(free_blocks))
74
79 - # Find all blocks with the lowest timestamp
80 - least_recent: List[PhysicalTokenBlock] = []
75 for block in free_blocks:
82 - if block.last_accessed == lowest_timestamp:
83 - least_recent.append(block)
84 -
85 - # Find highest prefix count per block
86 - highest_num_hashed_tokens = 0
87 - for block in least_recent:
88 - if block.num_hashed_tokens > highest_num_hashed_tokens:
89 - highest_num_hashed_tokens = block.num_hashed_tokens
90 -
91 - evicted_block: Optional[PhysicalTokenBlock] = None
92 -
93 - # Find the first block with the lowest timestamp
94 - for block in least_recent:
95 - if block.num_hashed_tokens == highest_num_hashed_tokens:
76 + if (block.last_accessed < evicted_block.last_accessed
77 + or block.last_accessed == evicted_block.last_accessed and
78 + block.num_hashed_tokens > evicted_block.num_hashed_tokens):
79 evicted_block = block
97 - break
98 -
99 - assert evicted_block is not None
80
81 del self.free_table[evicted_block.block_hash]
82
@@ -119,43 +99,8 @@ class LRUEvictor(Evictor):
99 return len(self.free_table)
100
101
122 -class RandomEvictor(Evictor):
123 - """Evicts in a first-in-first-out order"""
124 -
125 - def __init__(self):
126 - self.free_table: Dict[int, PhysicalTokenBlock] = {}
127 -
128 - def __contains__(self, block_hash: int) -> bool:
129 - return block_hash in self.free_table
130 -
131 - def evict(self) -> PhysicalTokenBlock:
132 - if len(self.free_table) == 0:
133 - raise ValueError("No usable cache memory left")
134 - evicted_block = next(iter(self.free_table.values()))
135 - evicted_block.computed = False
136 - del self.free_table[evicted_block.block_hash]
137 - return evicted_block
138 -
139 - def add(self, block: PhysicalTokenBlock):
140 - self.free_table[block.block_hash] = block
141 -
142 - def remove(self, block_hash: int) -> PhysicalTokenBlock:
143 - if block_hash not in self.free_table:
144 - raise ValueError(
145 - "Attempting to remove block that's not in the evictor")
146 - block: PhysicalTokenBlock = self.free_table[block_hash]
147 - del self.free_table[block_hash]
148 - return block
149 -
150 - @property
151 - def num_blocks(self) -> int:
152 - return len(self.free_table)
153 -
154 -
102 def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
103 if eviction_policy == EvictionPolicy.LRU:
104 return LRUEvictor()
158 - elif eviction_policy == EvictionPolicy.FIFO:
159 - return RandomEvictor()
105 else:
106 raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")