@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 201fc07730
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Prefix caching (take 2) (#9972)

Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>

Cody Yu committed Nov 7, 2024 at 17:34 UTC 201fc07730ec96dd88b758064f148a424f4b251b
6 files changed +771 -66
benchmarks/benchmark_prefix_caching.py
+1 -8
@@ -118,7 +118,7 @@ def main(args):
118 random.seed(args.seed)
119 if args.dataset_path is not None:
120 print(f"Start to sample {args.num_prompts} prompts"
121 - "from {args.dataset_path}")
121 + f"from {args.dataset_path}")
122 filtered_datasets = sample_requests(
123 dataset_path=args.dataset_path,
124 num_requests=args.num_prompts,
@@ -142,13 +142,6 @@ def main(args):
142 repeat_count=args.repeat_count,
143 sort=args.sort)
144
145 - print("------warm up------")
146 - test_prefix(
147 - llm=llm,
148 - prompts=prompts,
149 - sampling_params=sampling_params,
150 - )
151 -
145 print("------start generating------")
146 test_prefix(
147 llm=llm,
tests/v1/core/test_prefix_caching.py new
+219
@@ -0,0 +1,219 @@
1 +"""Compare the with and without prefix caching."""
2 +from vllm.inputs import DecoderOnlyInputs
3 +from vllm.sampling_params import SamplingParams
4 +from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
5 +from vllm.v1.core.kv_cache_utils import hash_block_tokens
6 +
7 +
8 +def make_request(request_id, prompt_token_ids):
9 + return Request(
10 + request_id=request_id,
11 + inputs=DecoderOnlyInputs(prompt_token_ids=prompt_token_ids),
12 + sampling_params=SamplingParams(max_tokens=17),
13 + eos_token_id=100,
14 + arrival_time=0,
15 + lora_request=None,
16 + )
17 +
18 +
19 +def test_prefill():
20 + manager = KVCacheManager(
21 + block_size=16,
22 + num_gpu_blocks=10,
23 + sliding_window=False,
24 + enable_caching=True,
25 + num_preallocate_tokens=16,
26 + )
27 +
28 + # Complete 3 blocks (48 tokens)
29 + common_token_ids = [i for i in range(3) for _ in range(16)]
30 +
31 + # Fully cache miss
32 + # Incomplete 1 block (7 tokens)
33 + unique_token_ids = [3] * 7
34 + req0 = make_request("0", common_token_ids + unique_token_ids)
35 + computed_blocks = manager.get_computed_blocks(req0)
36 + assert not computed_blocks
37 + blocks = manager.allocate_slots(req0, 55, computed_blocks)
38 + assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
39 +
40 + # Check full block metadata
41 + parent_block_hash = None
42 + for block_id in (0, 1, 2):
43 + block_hash = hash_block_tokens(parent_block_hash,
44 + manager.block_pool[block_id].token_ids)
45 + assert manager.block_pool[block_id].block_hash == block_hash
46 + assert manager.block_pool[block_id].ref_cnt == 1
47 + assert manager.block_pool[block_id].num_hashed_tokens == 16 * (
48 + block_id + 1)
49 + assert manager.block_pool[block_id].token_ids == tuple([block_id] * 16)
50 + parent_block_hash = block_hash
51 +
52 + # Check partial/preallocated block metadata
53 + for block_id in (3, 4):
54 + assert manager.block_pool[block_id].block_hash is None
55 + assert manager.block_pool[block_id].ref_cnt == 1
56 + assert manager.block_pool[block_id].num_hashed_tokens == 0
57 + if block_id == 3:
58 + assert manager.block_pool[block_id].token_ids == [3] * 7
59 + else:
60 + assert not manager.block_pool[block_id].token_ids
61 +
62 + # Cache hit in the common prefix when the original block is still in use.
63 + # Incomplete 1 block (5 tokens)
64 + unique_token_ids = [3] * 5
65 + req1 = make_request("1", common_token_ids + unique_token_ids)
66 + computed_blocks = manager.get_computed_blocks(req1)
67 + assert [b.block_id for b in computed_blocks] == [0, 1, 2]
68 + num_new_tokens = 53 - 3 * 16
69 + blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
70 + assert [b.block_id for b in blocks] == [5, 6]
71 + for block in computed_blocks:
72 + assert block.ref_cnt == 2
73 +
74 + # At this point, we should have 3 free blocks left.
75 + assert manager.free_block_queue.num_free_blocks == 3
76 +
77 + manager.free(req0)
78 + manager.free(req1)
79 +
80 + # All blocks should be available.
81 + assert manager.free_block_queue.num_free_blocks == 10
82 + # The order should be
83 + # [unallocated (7, 8)]
84 + # [unique_req0 (4, 3)]
85 + # [unique_req1 (6, 5)]
86 + # [common (2, 1, 0)]
87 + assert [
88 + b.block_id for b in manager.free_block_queue.get_all_free_blocks()
89 + ] == [7, 8, 9, 4, 3, 6, 5, 2, 1, 0]
90 +
91 + # Cache hit in the common prefix when the original block is already free.
92 + # Incomplete 1 block (6 tokens)
93 + unique_token_ids = [3] * 6
94 + req2 = make_request("2", common_token_ids + unique_token_ids)
95 + computed_block = manager.get_computed_blocks(req2)
96 + assert [b.block_id for b in computed_block] == [0, 1, 2]
97 + num_new_tokens = 53 - 3 * 16
98 + blocks = manager.allocate_slots(req2, num_new_tokens, computed_blocks)
99 + assert [b.block_id for b in blocks] == [7, 8]
100 +
101 + # Although we only have 5 free blocks, we have 8 blocks in
102 + # the free block queue due to lazy removal.
103 + assert manager.free_block_queue.num_free_blocks == 5
104 + assert all([
105 + b.ref_cnt == 0 for b in manager.free_block_queue.get_all_free_blocks()
106 + ])
107 + assert len([b
108 + for b in manager.free_block_queue.get_all_free_blocks()]) == 5
109 +
110 + manager.free(req2)
111 +
112 + # Cache miss and eviction.
113 + req3 = make_request("3", [99] * (16 * 9))
114 + computed_blocks = manager.get_computed_blocks(req3)
115 + assert not computed_blocks
116 + blocks = manager.allocate_slots(req2, 16 * 9, computed_blocks)
117 + # This block ID order also checks the eviction order.
118 + assert [b.block_id for b in blocks] == [9, 4, 3, 6, 5, 8, 7, 2, 1, 0]
119 + assert manager.free_block_queue.num_free_blocks == 0
120 + assert manager.free_block_queue.free_list_head is None
121 + assert manager.free_block_queue.free_list_tail is None
122 +
123 +
124 +def test_decode():
125 + manager = KVCacheManager(
126 + block_size=16,
127 + num_gpu_blocks=10,
128 + sliding_window=False,
129 + enable_caching=True,
130 + num_preallocate_tokens=16,
131 + )
132 +
133 + # Complete 3 blocks (48 tokens)
134 + common_token_ids = [i for i in range(3) for _ in range(16)]
135 +
136 + # Fully cache miss
137 + # Incomplete 1 block (7 tokens)
138 + unique_token_ids = [3] * 7
139 + req0 = make_request("0", common_token_ids + unique_token_ids)
140 + computed_blocks = manager.get_computed_blocks(req0)
141 + assert not computed_blocks
142 + blocks = manager.allocate_slots(req0, 55, computed_blocks)
143 + assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
144 +
145 + # Append slots without allocating a new block.
146 + req0.num_computed_tokens = 55
147 + for _ in range(4):
148 + req0.append_output_token_ids(8)
149 + new_blocks = manager.append_slots(req0, 4)
150 + assert new_blocks is not None and len(new_blocks) == 0
151 + assert len(manager.block_pool[3].token_ids) == 11
152 +
153 + # Append slots without allocating a new block, but start using the
154 + # preallocated block.
155 + req0.num_computed_tokens = 59
156 + # 6 tokens to fill the previous block, and 10 tokens to fill
157 + # the preallocated block.
158 + for _ in range(5 + 10):
159 + req0.append_output_token_ids(7)
160 + new_blocks = manager.append_slots(req0, 15)
161 + assert new_blocks is not None and len(new_blocks) == 0
162 + assert len(manager.block_pool[3].token_ids) == 16
163 + assert len(manager.block_pool[4].token_ids) == 10
164 +
165 + # Append slots with allocating a new block.
166 + req0.num_computed_tokens = 74
167 + # 6 tokens to fill the previous block, and 10 tokens to fill
168 + # the preallocated block.
169 + for _ in range(6 + 11):
170 + req0.append_output_token_ids(12)
171 + new_blocks = manager.append_slots(req0, 17)
172 + # Plus one preallocated block.
173 + assert new_blocks is not None and len(new_blocks) == 2
174 + assert len(manager.block_pool[4].token_ids) == 16
175 + assert len(manager.block_pool[5].token_ids) == 11
176 + assert len(manager.block_pool[6].token_ids) == 0
177 +
178 +
179 +def test_evict():
180 + manager = KVCacheManager(
181 + block_size=16,
182 + num_gpu_blocks=10,
183 + sliding_window=False,
184 + enable_caching=True,
185 + num_preallocate_tokens=16,
186 + )
187 +
188 + last_token_id = 5 * 16 + 7
189 + req0 = make_request("0", list(range(last_token_id)))
190 + computed_blocks = manager.get_computed_blocks(req0)
191 + assert not computed_blocks
192 + blocks = manager.allocate_slots(req0, 5 * 16 + 7, computed_blocks)
193 + assert len(blocks) == 7 # 5 full + 1 partial + 1 preallocated
194 +
195 + # 3 blocks.
196 + req1 = make_request("1", list(range(last_token_id,
197 + last_token_id + 3 * 16)))
198 + computed_blocks = manager.get_computed_blocks(req1)
199 + assert not computed_blocks
200 + blocks = manager.allocate_slots(req1, 3 * 16, computed_blocks)
201 + assert len(blocks) == 3 # 3 full blocks
202 + last_token_id += 3 * 16
203 +
204 + assert manager.free_block_queue.num_free_blocks == 0
205 +
206 + manager.free(req0)
207 + manager.free(req1)
208 + assert manager.free_block_queue.num_free_blocks == 10
209 + assert [
210 + b.block_id for b in manager.free_block_queue.get_all_free_blocks()
211 + ] == [6, 5, 4, 3, 2, 1, 0, 9, 8, 7]
212 +
213 + # Touch the first 2 blocks.
214 + req2 = make_request("2", list(range(2 * 16 + 3)))
215 + computed_blocks = manager.get_computed_blocks(req2)
216 + assert [b.block_id for b in computed_blocks] == [0, 1]
217 + blocks = manager.allocate_slots(req2, 3, computed_blocks)
218 + assert [b.block_id for b in blocks] == [6, 5]
219 + assert manager.free_block_queue.num_free_blocks == 6
vllm/v1/core/kv_cache_manager.py
+335 -47
@@ -1,9 +1,11 @@
1 +from collections import defaultdict
2 from typing import Dict, List, Optional
3
3 -import numpy as np
4 -
4 from vllm.logger import init_logger
5 from vllm.utils import cdiv
6 +from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
7 + KVCacheBlock, hash_block_tokens,
8 + hash_request_tokens)
9 from vllm.v1.request import Request
10
11 logger = init_logger(__name__)
@@ -36,73 +38,359 @@ class KVCacheManager:
38 self.num_preallocate_tokens = num_preallocate_tokens
39 self.num_preallocate_blocks = cdiv(num_preallocate_tokens, block_size)
40
39 - self.free_block_ids = list(range(num_gpu_blocks))
40 - self.req_to_block_ids: Dict[str, List[int]] = {}
41 - self.ref_cnts = np.zeros(num_gpu_blocks, dtype=np.int32)
41 + # A Block pool of all kv-cache blocks.
42 + self.block_pool: List[KVCacheBlock] = [
43 + KVCacheBlock(idx) for idx in range(num_gpu_blocks)
44 + ]
45 + # Free block queue that constructs and manipulates a doubly linked
46 + # list of free blocks (including eviction candidates when caching is
47 + # enabled).
48 + self.free_block_queue = FreeKVCacheBlockQueue(self.block_pool)
49 +
50 + # {block_hash: {block ID: block}}. A cached block is
51 + # a full block with a block hash that can be used for prefix caching.
52 + # The cached block may be used by running requests or in the
53 + # free_block_queue that could potentially be evicted.
54 + # NOTE: We currently don't de-duplicate the blocks in the cache,
55 + # meaning that if a block becomes full and is cached, we don't check
56 + # if there is already an identical block in the cache. This is because
57 + # we want to make sure the allocated block IDs won't change so that
58 + # block tables are append-only.
59 + self.cached_block_hash_to_block: Dict[BlockHashType, Dict[
60 + int, KVCacheBlock]] = defaultdict(dict)
61 +
62 + # Mapping from request ID to blocks to track the blocks allocated
63 + # for each request, so that we can free the blocks when the request
64 + # is finished.
65 + self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
66
43 - def get_computed_blocks(self, request: Request) -> List[int]:
67 + def get_computed_blocks(self, request: Request) -> List[KVCacheBlock]:
68 + """Get the computed (cached) blocks for the request.
69 + Note that the computed blocks must be full.
70 +
71 + Args:
72 + request: The request to get the computed blocks.
73 +
74 + Returns:
75 + A list of blocks that are computed for the request.
76 + """
77 if not self.enable_caching:
45 - # No prefix caching.
78 + # Prefix caching is disabled.
79 return []
47 - # TODO(woosuk): Implement hash-based caching.
48 - return []
80 +
81 + computed_blocks = []
82 + block_hashes = hash_request_tokens(self.block_size,
83 + request.all_token_ids)
84 +
85 + for block_hash in block_hashes:
86 + # block_hashes is a chain of block hashes. If a block hash is not
87 + # in the cached_block_hash_to_id, the following block hashes are
88 + # not computed yet for sure.
89 + if cached_block := self._get_cached_block(block_hash):
90 + computed_blocks.append(cached_block)
91 + else:
92 + break
93 +
94 + return computed_blocks
95
96 def append_slots(
97 self,
98 request: Request,
99 num_tokens: int,
54 - ) -> Optional[List[int]]:
100 + ) -> Optional[List[KVCacheBlock]]:
101 + """Append slots to the block table of the request.
102 + We first append slots to already allocated blocks. If the allocated
103 + blocks are not enough, we allocate new blocks.
104 +
105 + Args:
106 + request: The request to append slots.
107 + num_tokens: The number of tokens to append.
108 +
109 + Returns:
110 + A list of new blocks if new blocks are allocated, or None
111 + if new blocks are required but cannot be allocated.
112 + """
113 num_required_blocks = cdiv(request.num_computed_tokens + num_tokens,
114 self.block_size)
57 - req_block_ids = self.req_to_block_ids[request.request_id]
58 - if num_required_blocks <= len(req_block_ids):
59 - # No new block is needed.
60 - return []
115 + req_blocks = self.req_to_blocks[request.request_id]
116
62 - num_new_blocks = num_required_blocks - len(req_block_ids)
63 - num_free_blocks = len(self.free_block_ids)
64 - if num_new_blocks > num_free_blocks:
65 - # Cannot allocate new blocks.
117 + num_new_blocks = num_required_blocks - len(req_blocks)
118 + if num_new_blocks > self.free_block_queue.num_free_blocks:
119 + # Need to allocate new blocks due to insufficient pre-allocated
120 + # slots, but we cannot allocate new blocks due to the limit.
121 return None
122
68 - # Allocate new blocks.
123 + # When caching is enabled, assign token IDs to already allocated blocks.
124 + new_token_ids = None
125 + parent_block = None
126 + if self.enable_caching:
127 + # Figure out the token IDs to add to the blocks.
128 + new_token_ids = request.all_token_ids[
129 + request.num_computed_tokens:request.num_computed_tokens +
130 + num_tokens]
131 +
132 + # Find the last full block index.
133 + # TODO: This may be optimized by calculating the computed tokens.
134 + last_full_block_idx = len(req_blocks) - 1
135 + while (last_full_block_idx >= 0
136 + and req_blocks[last_full_block_idx].block_hash is None):
137 + last_full_block_idx -= 1
138 +
139 + parent_block = (req_blocks[last_full_block_idx]
140 + if last_full_block_idx >= 0 else None)
141 + token_id_idx = self._add_token_ids_to_blocks(
142 + blocks=req_blocks[last_full_block_idx + 1:],
143 + token_ids=new_token_ids,
144 + parent_block=parent_block)
145 +
146 + new_token_ids = new_token_ids[token_id_idx:]
147 + parent_block = req_blocks[-1]
148 +
149 + # No new block is needed. When caching is enabled, we make sure
150 + # token_id_idx is equal to len(new_token_ids), meaning that all tokens
151 + # are added to allocated blocks.
152 + if num_required_blocks <= len(req_blocks):
153 + assert not self.enable_caching or token_id_idx == num_tokens, \
154 + f"{token_id_idx=} != {num_tokens=}"
155 + return []
156 +
157 + # Allocate new blocks considering preallocated blocks, and
158 + # add token IDs to them if caching is enabled.
159 num_new_blocks = min(num_new_blocks + self.num_preallocate_blocks,
70 - num_free_blocks)
71 - new_block_ids = self._get_new_blocks(num_new_blocks)
72 - req_block_ids.extend(new_block_ids)
73 - self.ref_cnts[new_block_ids] += 1
74 - return new_block_ids
160 + self.free_block_queue.num_free_blocks)
161 + new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
162 + parent_block)
163 + req_blocks.extend(new_blocks)
164 + return new_blocks
165
166 def allocate_slots(
167 self,
168 request: Request,
169 num_tokens: int,
80 - computed_block_ids: List[int],
81 - ) -> Optional[List[int]]:
170 + computed_blocks: List[KVCacheBlock],
171 + ) -> Optional[List[KVCacheBlock]]:
172 + """Allocate slots for a new request.
173 +
174 + Args:
175 + request: The request to allocate slots.
176 + num_tokens: The number of tokens to allocate. Note that this does
177 + not include the tokens that have already been computed.
178 + computed_blocks: The blocks that have already been computed.
179 +
180 + Returns:
181 + A list of new allocated blocks.
182 + """
183 + if num_tokens == 0:
184 + raise ValueError(
185 + f"num_tokens must be greater than 0, got {num_tokens}")
186 +
187 + # If a computed block of a request is an eviction candidate (in the
188 + # free queue and ref_cnt == 0), it cannot be counted as a free block
189 + # when allocating this request.
190 + num_evictable_computed_blocks = len(
191 + [blk for blk in computed_blocks if blk.ref_cnt == 0])
192 +
193 num_required_blocks = cdiv(num_tokens, self.block_size)
83 - num_free_blocks = len(self.free_block_ids)
84 - if num_required_blocks > num_free_blocks:
194 + if (num_required_blocks > self.free_block_queue.num_free_blocks -
195 + num_evictable_computed_blocks):
196 # Cannot allocate new blocks.
197 return None
198
88 - num_new_blocks = min(num_required_blocks + self.num_preallocate_blocks,
89 - num_free_blocks)
90 - new_block_ids = self._get_new_blocks(num_new_blocks)
91 - block_ids = computed_block_ids + new_block_ids
92 - self.req_to_block_ids[request.request_id] = block_ids
93 - self.ref_cnts[block_ids] += 1
94 - return new_block_ids
199 + # Determine the number of new blocks to allocate considering
200 + # preallocated blocks.
201 + num_new_blocks = min(
202 + num_required_blocks + self.num_preallocate_blocks,
203 + self.free_block_queue.num_free_blocks -
204 + num_evictable_computed_blocks)
205 +
206 + num_computed_tokens = len(computed_blocks) * self.block_size
207 +
208 + # When caching is enabled, get the new token IDs and the parent block
209 + # ID to generate cache keys.
210 + new_token_ids = None
211 + parent_block = None
212 + if self.enable_caching:
213 + # Touch the computed blocks to make sure they won't be evicted.
214 + self._touch(computed_blocks)
215 +
216 + # Get the token IDs for the blocks being allocated for hashing.
217 + new_token_ids = request.all_token_ids[
218 + num_computed_tokens:num_computed_tokens + num_tokens]
219 + if not new_token_ids:
220 + raise RuntimeError(
221 + "Failed to infer the token IDs for allocation. "
222 + f"#all_tokens={len(request.all_token_ids)} < "
223 + f"#computed_tokens={num_computed_tokens}")
224 +
225 + # Get the parent block ID to construct the block chain.
226 + parent_block = computed_blocks[-1] if computed_blocks else None
227 +
228 + new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
229 + parent_block)
230 +
231 + # Concatenate the computed block IDs and the new block IDs.
232 + self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
233 + return new_blocks
234
235 def free(self, request: Request) -> None:
97 - block_ids = self.req_to_block_ids.pop(request.request_id)
98 - self.ref_cnts[block_ids] -= 1
99 - for block_id in block_ids:
100 - ref_cnt = self.ref_cnts[block_id]
101 - if ref_cnt == 0:
102 - self.free_block_ids.append(block_id)
103 -
104 - def _get_new_blocks(self, num_blocks: int) -> List[int]:
105 - assert num_blocks <= len(self.free_block_ids)
106 - new_block_ids = self.free_block_ids[-num_blocks:]
107 - self.free_block_ids = self.free_block_ids[:-num_blocks]
108 - return new_block_ids
236 + """Free the blocks allocated for the request.
237 + When caching is enabled, we free the blocks in reverse order so that
238 + the tail blocks are evicted first.
239 +
240 + Args:
241 + request: The request to free the blocks.
242 + """
243 + blocks = self.req_to_blocks.pop(request.request_id)
244 + if self.enable_caching:
245 + # Free blocks in reverse order so that the tail blocks are
246 + # freed first.
247 + blocks = reversed(blocks)
248 +
249 + for block in blocks:
250 + block.ref_cnt -= 1
251 + if block.ref_cnt == 0:
252 + self.free_block_queue.append(block)
253 +
254 + def _get_new_blocks(
255 + self,
256 + num_blocks: int,
257 + token_ids: Optional[List[int]] = None,
258 + parent_block: Optional[int] = None) -> List[KVCacheBlock]:
259 + """Get new blocks from the free block pool, and add token IDs to
260 + allocated blocks if caching is enabled.
261 + Note that we do not check block cache in this function.
262 +
263 + Args:
264 + num_blocks: The number of blocks to allocate.
265 + token_ids: The token IDs in the blocks. None if caching is disabled.
266 + parent_block: The parent block. Used to include block chain
267 + in the block hash.
268 +
269 + Returns:
270 + A list of new block.
271 + """
272 + if num_blocks > self.free_block_queue.num_free_blocks:
273 + raise ValueError(
274 + f"Cannot get {num_blocks} free blocks from the pool")
275 +
276 + # First allocate blocks.
277 + ret: List[KVCacheBlock] = []
278 + idx = 0
279 + while idx < num_blocks:
280 + curr_block = self.free_block_queue.popleft()
281 + assert curr_block.ref_cnt == 0
282 +
283 + # Evict blocks from the cache.
284 + if self.enable_caching:
285 + block_hash = curr_block.block_hash
286 + if (block_hash is not None
287 + and block_hash in self.cached_block_hash_to_block):
288 + if len(self.cached_block_hash_to_block[block_hash]) == 1:
289 + del self.cached_block_hash_to_block[block_hash]
290 + else:
291 + del self.cached_block_hash_to_block[block_hash][
292 + curr_block.block_id]
293 + curr_block.reset()
294 +
295 + curr_block.ref_cnt = 1
296 + ret.append(curr_block)
297 + idx += 1
298 +
299 + # Then assign token IDs to the allocated blocks.
300 + if self.enable_caching:
301 + assert token_ids is not None
302 + token_id_idx = self._add_token_ids_to_blocks(
303 + blocks=ret, token_ids=token_ids, parent_block=parent_block)
304 + assert token_id_idx == len(token_ids)
305 +
306 + return ret
307 +
308 + def _cache_full_block(self,
309 + block: KVCacheBlock,
310 + parent_block: Optional[KVCacheBlock] = None) -> None:
311 + """Cache a full block for prefix caching.
312 +
313 + Args:
314 + block: The block to cache.
315 + parent_block: The parent block. None if this is the first block.
316 + """
317 + parent_block_hash = (parent_block.block_hash
318 + if parent_block is not None else None)
319 + assert len(block.token_ids) == self.block_size
320 + block.token_ids = tuple(block.token_ids)
321 + block_hash = hash_block_tokens(parent_block_hash, block.token_ids)
322 + block.block_hash = block_hash
323 + block.num_hashed_tokens = self.block_size + (
324 + parent_block.num_hashed_tokens if parent_block is not None else 0)
325 + self.cached_block_hash_to_block[block_hash][block.block_id] = block
326 +
327 + def _get_cached_block(self,
328 + block_hash: BlockHashType) -> Optional[KVCacheBlock]:
329 + """Get a cached block by the block hash, or None if cache miss.
330 + If there are duplicated blocks, we return the first block in the cache.
331 +
332 + Args:
333 + block_hash: The hash value of the block.
334 +
335 + Returns:
336 + The cached block if it exists, or None.
337 + """
338 + if block_hash in self.cached_block_hash_to_block:
339 + first_block_id = list(
340 + self.cached_block_hash_to_block[block_hash].keys())[0]
341 + return self.cached_block_hash_to_block[block_hash][first_block_id]
342 + return None
343 +
344 + def _touch(self, blocks: List[KVCacheBlock]) -> None:
345 + """Touch a block increases its reference count by 1, and may remove
346 + the block from the free queue. This is used when a block is hit by
347 + another request with the same prefix.
348 +
349 + Args:
350 + blocks: A list of blocks to touch.
351 + """
352 + for block in blocks:
353 + # ref_cnt=0 means this block is in the free list (i.e. eviction
354 + # candidate), so remove it.
355 + if block.ref_cnt == 0:
356 + self.free_block_queue.remove(block)
357 + block.ref_cnt += 1
358 +
359 + def _add_token_ids_to_blocks(
360 + self,
361 + blocks: List[KVCacheBlock],
362 + token_ids: List[int],
363 + parent_block: Optional[KVCacheBlock] = None) -> int:
364 + """Add token IDs to a list of allocated blocks.
365 + If a block becomes full after adding token IDs, cache it.
366 + Return the token ID index that has not been added to the blocks
367 + if the blocks are not enough to hold all the token IDs.
368 +
369 + Args:
370 + blocks: A list of blocks to add token IDs.
371 + token_ids: A list of token IDs to add.
372 + parent_block: The parent block. None if this is the
373 + first block.
374 +
375 + Returns:
376 + The starting token ID index that has not been added to the blocks
377 + due to insufficient given blocks.
378 + """
379 + token_id_start = 0
380 + for curr_block in blocks:
381 + # If all token IDs are added, then the rest of the blocks are
382 + # preallocated blocks, so we only need to update the
383 + # parent_block_id. FIXME
384 + if token_id_start == len(token_ids):
385 + continue
386 +
387 + # Add token IDs to the empty slots in the block.
388 + empty_slots = self.block_size - len(curr_block.token_ids)
389 + token_id_end = min(token_id_start + empty_slots, len(token_ids))
390 + curr_block.token_ids.extend(token_ids[token_id_start:token_id_end])
391 + # Cache the block if it becomes full.
392 + if len(curr_block.token_ids) == self.block_size:
393 + self._cache_full_block(curr_block, parent_block)
394 + parent_block = curr_block
395 + token_id_start = token_id_end
396 + return token_id_start
vllm/v1/core/kv_cache_utils.py new
+194
@@ -0,0 +1,194 @@
1 +"""KV-Cache Utilities."""
2 +from dataclasses import dataclass, field
3 +from typing import List, Optional, Tuple, Union
4 +
5 +from vllm.logger import init_logger
6 +
7 +logger = init_logger(__name__)
8 +
9 +BlockHashType = Tuple[int, Tuple[int]]
10 +
11 +
12 +@dataclass
13 +class KVCacheBlock:
14 + """KV-cache block metadata."""
15 + # Block ID, ranging from 0 to num_gpu_blocks - 1.
16 + block_id: int
17 + # Reference count.
18 + ref_cnt: int = 0
19 + # Token IDs in the block. When the block is full, the type of token_ids
20 + # should be Tuple[int] for fast matching.
21 + token_ids: Union[List[int], Tuple[int]] = field(default_factory=list)
22 + # The hash of the block composed of (block hash, tuple of token IDs).
23 + # It is only available when the block is full.
24 + block_hash: Optional[BlockHashType] = None
25 + # The number of hashed tokens. More hashed tokens means the block
26 + # is closer to the end of a prompt and more likely to be evicted.
27 + num_hashed_tokens: int = 0
28 +
29 + # Used to construct a doubly linked list for free blocks.
30 + # These two attributes should only be manipulated by FreeKVCacheBlockQueue.
31 + prev_free_block: Optional["KVCacheBlock"] = None
32 + next_free_block: Optional["KVCacheBlock"] = None
33 +
34 + def reset(self):
35 + """Reset the block metadata."""
36 + self.ref_cnt = 0
37 + self.token_ids = []
38 + self.block_hash = None
39 + self.num_hashed_tokens = 0
40 +
41 +
42 +class FreeKVCacheBlockQueue:
43 + """This class organizes a list of KVCacheBlock objects to a doubly linked
44 + list of free blocks. We implement this class instead of using Python
45 + builtin deque to support removing a block in the middle of the queue
46 + in O(1) time. To close the performance gap to the builtin deque which is
47 + implemented in C++, this class does not allocate any Python objects when
48 + manipulating the linked list. Instead, this class manipulates the
49 + prev_free_block and next_free_block attributes of the given blocks.
50 +
51 + The queue is ordered by block ID in the beginning. When a block is allocated
52 + and then freed, it will be appended back with the eviction order:
53 + 1. The least recent used block is at the front (LRU).
54 + 2. If two blocks have the same last accessed time (allocated by the
55 + same sequence), the one with more hash tokens (the tail of a block
56 + chain) is at the front.
57 + Note that we maintain this order by reversing the block order when free
58 + blocks of a request. This operation is outside of this class.
59 +
60 + Args:
61 + blocks: A list of KVCacheBlock objects.
62 + """
63 +
64 + def __init__(self, blocks: List[KVCacheBlock]) -> None:
65 + self.num_free_blocks = len(blocks)
66 +
67 + # Initialize the doubly linked list of free blocks.
68 + self.free_list_head = blocks[0]
69 + self.free_list_tail = blocks[-1]
70 + for i in range(self.num_free_blocks):
71 + if i > 0:
72 + blocks[i].prev_free_block = blocks[i - 1]
73 + if i < self.num_free_blocks - 1:
74 + blocks[i].next_free_block = blocks[i + 1]
75 +
76 + def popleft(self) -> KVCacheBlock:
77 + """Pop the first free block and reduce num_free_blocks by 1.
78 +
79 + Returns:
80 + The first free block.
81 + """
82 + if not self.free_list_head:
83 + raise ValueError("No free blocks available")
84 +
85 + block = self.free_list_head
86 + self.remove(block)
87 + return block
88 +
89 + def remove(self, block: KVCacheBlock) -> None:
90 + """Remove a block in the free list and reduce num_free_blocks by 1.
91 +
92 + Args:
93 + block: The block to remove.
94 + """
95 + if block.prev_free_block is not None:
96 + # Link the previous block to the next block.
97 + block.prev_free_block.next_free_block = block.next_free_block
98 + if block.next_free_block is not None:
99 + # Link the next block to the previous block.
100 + block.next_free_block.prev_free_block = block.prev_free_block
101 +
102 + if block == self.free_list_head:
103 + # Update the head if the block is the head.
104 + self.free_list_head = block.next_free_block
105 + if block == self.free_list_tail:
106 + # Update the tail if the block is the tail.
107 + self.free_list_tail = block.prev_free_block
108 +
109 + # Remove the block from the linked list.
110 + block.prev_free_block = block.next_free_block = None
111 + self.num_free_blocks -= 1
112 +
113 + def append(self, block: KVCacheBlock) -> None:
114 + """Put a block back into the free list and increase
115 + num_free_blocks by 1.
116 +
117 + Args:
118 + block: The block to append.
119 + """
120 + if self.free_list_tail is not None:
121 + # Link the last block to the new block.
122 + self.free_list_tail.next_free_block = block
123 + block.prev_free_block = self.free_list_tail
124 + self.free_list_tail = block
125 + else:
126 + # The free list is empty.
127 + assert self.free_list_head is None
128 + self.free_list_head = self.free_list_tail = block
129 +
130 + block.next_free_block = None
131 + self.num_free_blocks += 1
132 +
133 + def get_all_free_blocks(self) -> List[KVCacheBlock]:
134 + """Get all free blocks in the free list. Mainly used for testing.
135 +
136 + Returns:
137 + A list of free blocks.
138 + """
139 + ret = []
140 + curr_block = self.free_list_head
141 + while curr_block is not None:
142 + ret.append(curr_block)
143 + curr_block = curr_block.next_free_block
144 + return ret
145 +
146 +
147 +def hash_block_tokens(parent_block_hash: Optional[int],
148 + curr_block_token_ids: Tuple[int]) -> BlockHashType:
149 + """Computes a hash value corresponding to the contents of a block and
150 + the contents of the preceding block(s). The hash value is used for
151 + prefix caching. We use LRU cache for this function to avoid recomputing
152 + hash values for the same block contents.
153 +
154 + TODO: Support arbitrary metadata so that we could support more
155 + features such as LoRA adapter.
156 +
157 + Args:
158 + parent_block_hash: The hash of the parent block. None
159 + if this is the first block.
160 + curr_block_token_ids: A tuple of token ids in the current
161 + block. The current block is assumed to be full.
162 +
163 + Returns:
164 + The hash value of the block and the token ids in the block.
165 + The entire tuple is used as the hash key of the block.
166 + """
167 + return (hash(
168 + (parent_block_hash, *curr_block_token_ids)), curr_block_token_ids)
169 +
170 +
171 +def hash_request_tokens(block_size: int,
172 + token_ids: List[int]) -> List[BlockHashType]:
173 + """Computes hash values of a chain of blocks given a sequence of
174 + token IDs. The hash value is used for prefix caching.
175 +
176 + Args:
177 + block_size: The size of each block.
178 + token_ids: A sequence of token ids in the request.
179 +
180 + Returns:
181 + The list of computed hash values.
182 + """
183 + ret = []
184 + parent_block_hash = None
185 + for start in range(0, len(token_ids), block_size):
186 + end = start + block_size
187 + block_token_ids = tuple(token_ids[start:end])
188 + # Do not hash the block if it is not full.
189 + if len(block_token_ids) < block_size:
190 + break
191 + block_hash = hash_block_tokens(parent_block_hash, block_token_ids)
192 + ret.append(block_hash)
193 + parent_block_hash = block_hash
194 + return ret
vllm/v1/core/scheduler.py
+21 -11
@@ -34,7 +34,7 @@ class Scheduler:
34 block_size=self.cache_config.block_size,
35 num_gpu_blocks=num_gpu_blocks,
36 sliding_window=self.cache_config.sliding_window,
37 - enable_caching=True)
37 + enable_caching=self.cache_config.enable_prefix_caching)
38 self.block_size = self.cache_config.block_size
39
40 # Scheduling constraints.
@@ -91,9 +91,9 @@ class Scheduler:
91 assert num_new_tokens > 0
92
93 while True:
94 - new_block_ids = self.kv_cache_manager.append_slots(
94 + new_blocks = self.kv_cache_manager.append_slots(
95 request, num_new_tokens)
96 - if new_block_ids is None:
96 + if new_blocks is None:
97 # The request cannot be scheduled.
98 # Preempt the lowest-priority request.
99 preempted_req = self.running.pop()
@@ -110,7 +110,9 @@ class Scheduler:
110 # The request can be scheduled.
111 scheduled_running_reqs.append(request)
112
113 - req_to_new_block_ids[request.request_id] = new_block_ids
113 + req_to_new_block_ids[request.request_id] = [
114 + b.block_id for b in new_blocks
115 + ]
116 num_scheduled_tokens[request.request_id] = num_new_tokens
117 token_budget -= num_new_tokens
118 req_index += 1
@@ -126,22 +128,29 @@ class Scheduler:
128
129 request = self.waiting[0]
130 # Get already-cached tokens.
129 - computed_block_ids = self.kv_cache_manager.get_computed_blocks(
131 + computed_blocks = self.kv_cache_manager.get_computed_blocks(
132 request)
133 # NOTE(woosuk): Since incomplete blocks are not eligible for
134 # sharing, `num_computed_tokens` is always a multiple of
135 # `block_size`.
134 - num_computed_tokens = len(computed_block_ids) * self.block_size
136 + num_computed_tokens = len(computed_blocks) * self.block_size
137 # Number of tokens to be scheduled.
138 # We use `request.num_tokens` instead of
139 # `request.num_prompt_tokens` to consider the resumed requests,
140 # which have output tokens.
141 num_new_tokens = request.num_tokens - num_computed_tokens
142 + if num_new_tokens == 0:
143 + # The happens when prompt length is divisible by the block
144 + # size and all blocks are cached. Now we force to recompute
145 + # the last token.
146 + num_computed_tokens -= 1
147 + num_new_tokens = 1
148 + computed_blocks.pop()
149 num_new_tokens = min(num_new_tokens, token_budget)
150 assert num_new_tokens > 0
142 - new_block_ids = self.kv_cache_manager.allocate_slots(
143 - request, num_new_tokens, computed_block_ids)
144 - if new_block_ids is None:
151 + new_blocks = self.kv_cache_manager.allocate_slots(
152 + request, num_new_tokens, computed_blocks)
153 + if new_blocks is None:
154 # The request cannot be scheduled.
155 break
156 request.num_computed_tokens = num_computed_tokens
@@ -156,8 +165,9 @@ class Scheduler:
165 raise RuntimeError(
166 f"Invalid request status: {request.status}")
167
159 - req_to_new_block_ids[request.request_id] = (
160 - computed_block_ids + new_block_ids)
168 + req_to_new_block_ids[request.request_id] = [
169 + b.block_id for b in computed_blocks + new_blocks
170 + ]
171 num_scheduled_tokens[request.request_id] = num_new_tokens
172 token_budget -= num_new_tokens
173 request.status = RequestStatus.RUNNING
vllm/v1/engine/llm_engine.py
+1
@@ -65,6 +65,7 @@ class LLMEngine:
65 elif usage_context == UsageContext.OPENAI_API_SERVER:
66 scheduler_config.max_num_seqs = 1024
67 scheduler_config.max_num_batched_tokens = 2048
68 + cache_config.enable_prefix_caching = True
69
70 logger.info(
71 "Initializing an LLM engine (v%s) with config: "