A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Prefix caching (take 2) (#9972)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
Cody Yu committed
Nov 7, 2024 at 17:34 UTC
201fc07730ec96dd88b758064f148a424f4b251b
6 files changed
+771
-66
benchmarks/benchmark_prefix_caching.py
+1
-8
@@ -118,7 +118,7 @@ def main(args):
118
random.seed(args.seed)
119
if args.dataset_path is not None:
120
print(f"Start to sample {args.num_prompts} prompts"
121
- "from {args.dataset_path}")
121
+ f"from {args.dataset_path}")
122
filtered_datasets = sample_requests(
123
dataset_path=args.dataset_path,
124
num_requests=args.num_prompts,
@@ -142,13 +142,6 @@ def main(args):
142
repeat_count=args.repeat_count,
143
sort=args.sort)
144
145
- print("------warm up------")
146
- test_prefix(
147
- llm=llm,
148
- prompts=prompts,
149
- sampling_params=sampling_params,
150
- )
151
-
145
print("------start generating------")
146
test_prefix(
147
llm=llm,
tests/v1/core/test_prefix_caching.py
new
+219
@@ -0,0 +1,219 @@
1
+"""Compare the with and without prefix caching."""
2
+from vllm.inputs import DecoderOnlyInputs
3
+from vllm.sampling_params import SamplingParams
4
+from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
5
+from vllm.v1.core.kv_cache_utils import hash_block_tokens
6
+
7
+
8
+def make_request(request_id, prompt_token_ids):
9
+ return Request(
10
+ request_id=request_id,
11
+ inputs=DecoderOnlyInputs(prompt_token_ids=prompt_token_ids),
12
+ sampling_params=SamplingParams(max_tokens=17),
13
+ eos_token_id=100,
14
+ arrival_time=0,
15
+ lora_request=None,
16
+ )
17
+
18
+
19
+def test_prefill():
20
+ manager = KVCacheManager(
21
+ block_size=16,
22
+ num_gpu_blocks=10,
23
+ sliding_window=False,
24
+ enable_caching=True,
25
+ num_preallocate_tokens=16,
26
+ )
27
+
28
+ # Complete 3 blocks (48 tokens)
29
+ common_token_ids = [i for i in range(3) for _ in range(16)]
30
+
31
+ # Fully cache miss
32
+ # Incomplete 1 block (7 tokens)
33
+ unique_token_ids = [3] * 7
34
+ req0 = make_request("0", common_token_ids + unique_token_ids)
35
+ computed_blocks = manager.get_computed_blocks(req0)
36
+ assert not computed_blocks
37
+ blocks = manager.allocate_slots(req0, 55, computed_blocks)
38
+ assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
39
+
40
+ # Check full block metadata
41
+ parent_block_hash = None
42
+ for block_id in (0, 1, 2):
43
+ block_hash = hash_block_tokens(parent_block_hash,
44
+ manager.block_pool[block_id].token_ids)
45
+ assert manager.block_pool[block_id].block_hash == block_hash
46
+ assert manager.block_pool[block_id].ref_cnt == 1
47
+ assert manager.block_pool[block_id].num_hashed_tokens == 16 * (
48
+ block_id + 1)
49
+ assert manager.block_pool[block_id].token_ids == tuple([block_id] * 16)
50
+ parent_block_hash = block_hash
51
+
52
+ # Check partial/preallocated block metadata
53
+ for block_id in (3, 4):
54
+ assert manager.block_pool[block_id].block_hash is None
55
+ assert manager.block_pool[block_id].ref_cnt == 1
56
+ assert manager.block_pool[block_id].num_hashed_tokens == 0
57
+ if block_id == 3:
58
+ assert manager.block_pool[block_id].token_ids == [3] * 7
59
+ else:
60
+ assert not manager.block_pool[block_id].token_ids
61
+
62
+ # Cache hit in the common prefix when the original block is still in use.
63
+ # Incomplete 1 block (5 tokens)
64
+ unique_token_ids = [3] * 5
65
+ req1 = make_request("1", common_token_ids + unique_token_ids)
66
+ computed_blocks = manager.get_computed_blocks(req1)
67
+ assert [b.block_id for b in computed_blocks] == [0, 1, 2]
68
+ num_new_tokens = 53 - 3 * 16
69
+ blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
70
+ assert [b.block_id for b in blocks] == [5, 6]
71
+ for block in computed_blocks:
72
+ assert block.ref_cnt == 2
73
+
74
+ # At this point, we should have 3 free blocks left.
75
+ assert manager.free_block_queue.num_free_blocks == 3
76
+
77
+ manager.free(req0)
78
+ manager.free(req1)
79
+
80
+ # All blocks should be available.
81
+ assert manager.free_block_queue.num_free_blocks == 10
82
+ # The order should be
83
+ # [unallocated (7, 8)]
84
+ # [unique_req0 (4, 3)]
85
+ # [unique_req1 (6, 5)]
86
+ # [common (2, 1, 0)]
87
+ assert [
88
+ b.block_id for b in manager.free_block_queue.get_all_free_blocks()
89
+ ] == [7, 8, 9, 4, 3, 6, 5, 2, 1, 0]
90
+
91
+ # Cache hit in the common prefix when the original block is already free.
92
+ # Incomplete 1 block (6 tokens)
93
+ unique_token_ids = [3] * 6
94
+ req2 = make_request("2", common_token_ids + unique_token_ids)
95
+ computed_block = manager.get_computed_blocks(req2)
96
+ assert [b.block_id for b in computed_block] == [0, 1, 2]
97
+ num_new_tokens = 53 - 3 * 16
98
+ blocks = manager.allocate_slots(req2, num_new_tokens, computed_blocks)
99
+ assert [b.block_id for b in blocks] == [7, 8]
100
+
101
+ # Although we only have 5 free blocks, we have 8 blocks in
102
+ # the free block queue due to lazy removal.
103
+ assert manager.free_block_queue.num_free_blocks == 5
104
+ assert all([
105
+ b.ref_cnt == 0 for b in manager.free_block_queue.get_all_free_blocks()
106
+ ])
107
+ assert len([b
108
+ for b in manager.free_block_queue.get_all_free_blocks()]) == 5
109
+
110
+ manager.free(req2)
111
+
112
+ # Cache miss and eviction.
113
+ req3 = make_request("3", [99] * (16 * 9))
114
+ computed_blocks = manager.get_computed_blocks(req3)
115
+ assert not computed_blocks
116
+ blocks = manager.allocate_slots(req2, 16 * 9, computed_blocks)
117
+ # This block ID order also checks the eviction order.
118
+ assert [b.block_id for b in blocks] == [9, 4, 3, 6, 5, 8, 7, 2, 1, 0]
119
+ assert manager.free_block_queue.num_free_blocks == 0
120
+ assert manager.free_block_queue.free_list_head is None
121
+ assert manager.free_block_queue.free_list_tail is None
122
+
123
+
124
+def test_decode():
125
+ manager = KVCacheManager(
126
+ block_size=16,
127
+ num_gpu_blocks=10,
128
+ sliding_window=False,
129
+ enable_caching=True,
130
+ num_preallocate_tokens=16,
131
+ )
132
+
133
+ # Complete 3 blocks (48 tokens)
134
+ common_token_ids = [i for i in range(3) for _ in range(16)]
135
+
136
+ # Fully cache miss
137
+ # Incomplete 1 block (7 tokens)
138
+ unique_token_ids = [3] * 7
139
+ req0 = make_request("0", common_token_ids + unique_token_ids)
140
+ computed_blocks = manager.get_computed_blocks(req0)
141
+ assert not computed_blocks
142
+ blocks = manager.allocate_slots(req0, 55, computed_blocks)
143
+ assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
144
+
145
+ # Append slots without allocating a new block.
146
+ req0.num_computed_tokens = 55
147
+ for _ in range(4):
148
+ req0.append_output_token_ids(8)
149
+ new_blocks = manager.append_slots(req0, 4)
150
+ assert new_blocks is not None and len(new_blocks) == 0
151
+ assert len(manager.block_pool[3].token_ids) == 11
152
+
153
+ # Append slots without allocating a new block, but start using the
154
+ # preallocated block.
155
+ req0.num_computed_tokens = 59
156
+ # 6 tokens to fill the previous block, and 10 tokens to fill
157
+ # the preallocated block.
158
+ for _ in range(5 + 10):
159
+ req0.append_output_token_ids(7)
160
+ new_blocks = manager.append_slots(req0, 15)
161
+ assert new_blocks is not None and len(new_blocks) == 0
162
+ assert len(manager.block_pool[3].token_ids) == 16
163
+ assert len(manager.block_pool[4].token_ids) == 10
164
+
165
+ # Append slots with allocating a new block.
166
+ req0.num_computed_tokens = 74
167
+ # 6 tokens to fill the previous block, and 10 tokens to fill
168
+ # the preallocated block.
169
+ for _ in range(6 + 11):
170
+ req0.append_output_token_ids(12)
171
+ new_blocks = manager.append_slots(req0, 17)
172
+ # Plus one preallocated block.
173
+ assert new_blocks is not None and len(new_blocks) == 2
174
+ assert len(manager.block_pool[4].token_ids) == 16
175
+ assert len(manager.block_pool[5].token_ids) == 11
176
+ assert len(manager.block_pool[6].token_ids) == 0
177
+
178
+
179
+def test_evict():
180
+ manager = KVCacheManager(
181
+ block_size=16,
182
+ num_gpu_blocks=10,
183
+ sliding_window=False,
184
+ enable_caching=True,
185
+ num_preallocate_tokens=16,
186
+ )
187
+
188
+ last_token_id = 5 * 16 + 7
189
+ req0 = make_request("0", list(range(last_token_id)))
190
+ computed_blocks = manager.get_computed_blocks(req0)
191
+ assert not computed_blocks
192
+ blocks = manager.allocate_slots(req0, 5 * 16 + 7, computed_blocks)
193
+ assert len(blocks) == 7 # 5 full + 1 partial + 1 preallocated
194
+
195
+ # 3 blocks.
196
+ req1 = make_request("1", list(range(last_token_id,
197
+ last_token_id + 3 * 16)))
198
+ computed_blocks = manager.get_computed_blocks(req1)
199
+ assert not computed_blocks
200
+ blocks = manager.allocate_slots(req1, 3 * 16, computed_blocks)
201
+ assert len(blocks) == 3 # 3 full blocks
202
+ last_token_id += 3 * 16
203
+
204
+ assert manager.free_block_queue.num_free_blocks == 0
205
+
206
+ manager.free(req0)
207
+ manager.free(req1)
208
+ assert manager.free_block_queue.num_free_blocks == 10
209
+ assert [
210
+ b.block_id for b in manager.free_block_queue.get_all_free_blocks()
211
+ ] == [6, 5, 4, 3, 2, 1, 0, 9, 8, 7]
212
+
213
+ # Touch the first 2 blocks.
214
+ req2 = make_request("2", list(range(2 * 16 + 3)))
215
+ computed_blocks = manager.get_computed_blocks(req2)
216
+ assert [b.block_id for b in computed_blocks] == [0, 1]
217
+ blocks = manager.allocate_slots(req2, 3, computed_blocks)
218
+ assert [b.block_id for b in blocks] == [6, 5]
219
+ assert manager.free_block_queue.num_free_blocks == 6
vllm/v1/core/kv_cache_manager.py
+335
-47
@@ -1,9 +1,11 @@
1
+from collections import defaultdict
2
from typing import Dict, List, Optional
3
3
-import numpy as np
4
-
4
from vllm.logger import init_logger
5
from vllm.utils import cdiv
6
+from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
7
+ KVCacheBlock, hash_block_tokens,
8
+ hash_request_tokens)
9
from vllm.v1.request import Request
10
11
logger = init_logger(__name__)
@@ -36,73 +38,359 @@ class KVCacheManager:
38
self.num_preallocate_tokens = num_preallocate_tokens
39
self.num_preallocate_blocks = cdiv(num_preallocate_tokens, block_size)
40
39
- self.free_block_ids = list(range(num_gpu_blocks))
40
- self.req_to_block_ids: Dict[str, List[int]] = {}
41
- self.ref_cnts = np.zeros(num_gpu_blocks, dtype=np.int32)
41
+ # A Block pool of all kv-cache blocks.
42
+ self.block_pool: List[KVCacheBlock] = [
43
+ KVCacheBlock(idx) for idx in range(num_gpu_blocks)
44
+ ]
45
+ # Free block queue that constructs and manipulates a doubly linked
46
+ # list of free blocks (including eviction candidates when caching is
47
+ # enabled).
48
+ self.free_block_queue = FreeKVCacheBlockQueue(self.block_pool)
49
+
50
+ # {block_hash: {block ID: block}}. A cached block is
51
+ # a full block with a block hash that can be used for prefix caching.
52
+ # The cached block may be used by running requests or in the
53
+ # free_block_queue that could potentially be evicted.
54
+ # NOTE: We currently don't de-duplicate the blocks in the cache,
55
+ # meaning that if a block becomes full and is cached, we don't check
56
+ # if there is already an identical block in the cache. This is because
57
+ # we want to make sure the allocated block IDs won't change so that
58
+ # block tables are append-only.
59
+ self.cached_block_hash_to_block: Dict[BlockHashType, Dict[
60
+ int, KVCacheBlock]] = defaultdict(dict)
61
+
62
+ # Mapping from request ID to blocks to track the blocks allocated
63
+ # for each request, so that we can free the blocks when the request
64
+ # is finished.
65
+ self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
66
43
- def get_computed_blocks(self, request: Request) -> List[int]:
67
+ def get_computed_blocks(self, request: Request) -> List[KVCacheBlock]:
68
+ """Get the computed (cached) blocks for the request.
69
+ Note that the computed blocks must be full.
70
+
71
+ Args:
72
+ request: The request to get the computed blocks.
73
+
74
+ Returns:
75
+ A list of blocks that are computed for the request.
76
+ """
77
if not self.enable_caching:
45
- # No prefix caching.
78
+ # Prefix caching is disabled.
79
return []
47
- # TODO(woosuk): Implement hash-based caching.
48
- return []
80
+
81
+ computed_blocks = []
82
+ block_hashes = hash_request_tokens(self.block_size,
83
+ request.all_token_ids)
84
+
85
+ for block_hash in block_hashes:
86
+ # block_hashes is a chain of block hashes. If a block hash is not
87
+ # in the cached_block_hash_to_id, the following block hashes are
88
+ # not computed yet for sure.
89
+ if cached_block := self._get_cached_block(block_hash):
90
+ computed_blocks.append(cached_block)
91
+ else:
92
+ break
93
+
94
+ return computed_blocks
95
96
def append_slots(
97
self,
98
request: Request,
99
num_tokens: int,
54
- ) -> Optional[List[int]]:
100
+ ) -> Optional[List[KVCacheBlock]]:
101
+ """Append slots to the block table of the request.
102
+ We first append slots to already allocated blocks. If the allocated
103
+ blocks are not enough, we allocate new blocks.
104
+
105
+ Args:
106
+ request: The request to append slots.
107
+ num_tokens: The number of tokens to append.
108
+
109
+ Returns:
110
+ A list of new blocks if new blocks are allocated, or None
111
+ if new blocks are required but cannot be allocated.
112
+ """
113
num_required_blocks = cdiv(request.num_computed_tokens + num_tokens,
114
self.block_size)
57
- req_block_ids = self.req_to_block_ids[request.request_id]
58
- if num_required_blocks <= len(req_block_ids):
59
- # No new block is needed.
60
- return []
115
+ req_blocks = self.req_to_blocks[request.request_id]
116
62
- num_new_blocks = num_required_blocks - len(req_block_ids)
63
- num_free_blocks = len(self.free_block_ids)
64
- if num_new_blocks > num_free_blocks:
65
- # Cannot allocate new blocks.
117
+ num_new_blocks = num_required_blocks - len(req_blocks)
118
+ if num_new_blocks > self.free_block_queue.num_free_blocks:
119
+ # Need to allocate new blocks due to insufficient pre-allocated
120
+ # slots, but we cannot allocate new blocks due to the limit.
121
return None
122
68
- # Allocate new blocks.
123
+ # When caching is enabled, assign token IDs to already allocated blocks.
124
+ new_token_ids = None
125
+ parent_block = None
126
+ if self.enable_caching:
127
+ # Figure out the token IDs to add to the blocks.
128
+ new_token_ids = request.all_token_ids[
129
+ request.num_computed_tokens:request.num_computed_tokens +
130
+ num_tokens]
131
+
132
+ # Find the last full block index.
133
+ # TODO: This may be optimized by calculating the computed tokens.
134
+ last_full_block_idx = len(req_blocks) - 1
135
+ while (last_full_block_idx >= 0
136
+ and req_blocks[last_full_block_idx].block_hash is None):
137
+ last_full_block_idx -= 1
138
+
139
+ parent_block = (req_blocks[last_full_block_idx]
140
+ if last_full_block_idx >= 0 else None)
141
+ token_id_idx = self._add_token_ids_to_blocks(
142
+ blocks=req_blocks[last_full_block_idx + 1:],
143
+ token_ids=new_token_ids,
144
+ parent_block=parent_block)
145
+
146
+ new_token_ids = new_token_ids[token_id_idx:]
147
+ parent_block = req_blocks[-1]
148
+
149
+ # No new block is needed. When caching is enabled, we make sure
150
+ # token_id_idx is equal to len(new_token_ids), meaning that all tokens
151
+ # are added to allocated blocks.
152
+ if num_required_blocks <= len(req_blocks):
153
+ assert not self.enable_caching or token_id_idx == num_tokens, \
154
+ f"{token_id_idx=} != {num_tokens=}"
155
+ return []
156
+
157
+ # Allocate new blocks considering preallocated blocks, and
158
+ # add token IDs to them if caching is enabled.
159
num_new_blocks = min(num_new_blocks + self.num_preallocate_blocks,
70
- num_free_blocks)
71
- new_block_ids = self._get_new_blocks(num_new_blocks)
72
- req_block_ids.extend(new_block_ids)
73
- self.ref_cnts[new_block_ids] += 1
74
- return new_block_ids
160
+ self.free_block_queue.num_free_blocks)
161
+ new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
162
+ parent_block)
163
+ req_blocks.extend(new_blocks)
164
+ return new_blocks
165
166
def allocate_slots(
167
self,
168
request: Request,
169
num_tokens: int,
80
- computed_block_ids: List[int],
81
- ) -> Optional[List[int]]:
170
+ computed_blocks: List[KVCacheBlock],
171
+ ) -> Optional[List[KVCacheBlock]]:
172
+ """Allocate slots for a new request.
173
+
174
+ Args:
175
+ request: The request to allocate slots.
176
+ num_tokens: The number of tokens to allocate. Note that this does
177
+ not include the tokens that have already been computed.
178
+ computed_blocks: The blocks that have already been computed.
179
+
180
+ Returns:
181
+ A list of new allocated blocks.
182
+ """
183
+ if num_tokens == 0:
184
+ raise ValueError(
185
+ f"num_tokens must be greater than 0, got {num_tokens}")
186
+
187
+ # If a computed block of a request is an eviction candidate (in the
188
+ # free queue and ref_cnt == 0), it cannot be counted as a free block
189
+ # when allocating this request.
190
+ num_evictable_computed_blocks = len(
191
+ [blk for blk in computed_blocks if blk.ref_cnt == 0])
192
+
193
num_required_blocks = cdiv(num_tokens, self.block_size)
83
- num_free_blocks = len(self.free_block_ids)
84
- if num_required_blocks > num_free_blocks:
194
+ if (num_required_blocks > self.free_block_queue.num_free_blocks -
195
+ num_evictable_computed_blocks):
196
# Cannot allocate new blocks.
197
return None
198
88
- num_new_blocks = min(num_required_blocks + self.num_preallocate_blocks,
89
- num_free_blocks)
90
- new_block_ids = self._get_new_blocks(num_new_blocks)
91
- block_ids = computed_block_ids + new_block_ids
92
- self.req_to_block_ids[request.request_id] = block_ids
93
- self.ref_cnts[block_ids] += 1
94
- return new_block_ids
199
+ # Determine the number of new blocks to allocate considering
200
+ # preallocated blocks.
201
+ num_new_blocks = min(
202
+ num_required_blocks + self.num_preallocate_blocks,
203
+ self.free_block_queue.num_free_blocks -
204
+ num_evictable_computed_blocks)
205
+
206
+ num_computed_tokens = len(computed_blocks) * self.block_size
207
+
208
+ # When caching is enabled, get the new token IDs and the parent block
209
+ # ID to generate cache keys.
210
+ new_token_ids = None
211
+ parent_block = None
212
+ if self.enable_caching:
213
+ # Touch the computed blocks to make sure they won't be evicted.
214
+ self._touch(computed_blocks)
215
+
216
+ # Get the token IDs for the blocks being allocated for hashing.
217
+ new_token_ids = request.all_token_ids[
218
+ num_computed_tokens:num_computed_tokens + num_tokens]
219
+ if not new_token_ids:
220
+ raise RuntimeError(
221
+ "Failed to infer the token IDs for allocation. "
222
+ f"#all_tokens={len(request.all_token_ids)} < "
223
+ f"#computed_tokens={num_computed_tokens}")
224
+
225
+ # Get the parent block ID to construct the block chain.
226
+ parent_block = computed_blocks[-1] if computed_blocks else None
227
+
228
+ new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
229
+ parent_block)
230
+
231
+ # Concatenate the computed block IDs and the new block IDs.
232
+ self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
233
+ return new_blocks
234
235
def free(self, request: Request) -> None:
97
- block_ids = self.req_to_block_ids.pop(request.request_id)
98
- self.ref_cnts[block_ids] -= 1
99
- for block_id in block_ids:
100
- ref_cnt = self.ref_cnts[block_id]
101
- if ref_cnt == 0:
102
- self.free_block_ids.append(block_id)
103
-
104
- def _get_new_blocks(self, num_blocks: int) -> List[int]:
105
- assert num_blocks <= len(self.free_block_ids)
106
- new_block_ids = self.free_block_ids[-num_blocks:]
107
- self.free_block_ids = self.free_block_ids[:-num_blocks]
108
- return new_block_ids
236
+ """Free the blocks allocated for the request.
237
+ When caching is enabled, we free the blocks in reverse order so that
238
+ the tail blocks are evicted first.
239
+
240
+ Args:
241
+ request: The request to free the blocks.
242
+ """
243
+ blocks = self.req_to_blocks.pop(request.request_id)
244
+ if self.enable_caching:
245
+ # Free blocks in reverse order so that the tail blocks are
246
+ # freed first.
247
+ blocks = reversed(blocks)
248
+
249
+ for block in blocks:
250
+ block.ref_cnt -= 1
251
+ if block.ref_cnt == 0:
252
+ self.free_block_queue.append(block)
253
+
254
+ def _get_new_blocks(
255
+ self,
256
+ num_blocks: int,
257
+ token_ids: Optional[List[int]] = None,
258
+ parent_block: Optional[int] = None) -> List[KVCacheBlock]:
259
+ """Get new blocks from the free block pool, and add token IDs to
260
+ allocated blocks if caching is enabled.
261
+ Note that we do not check block cache in this function.
262
+
263
+ Args:
264
+ num_blocks: The number of blocks to allocate.
265
+ token_ids: The token IDs in the blocks. None if caching is disabled.
266
+ parent_block: The parent block. Used to include block chain
267
+ in the block hash.
268
+
269
+ Returns:
270
+ A list of new block.
271
+ """
272
+ if num_blocks > self.free_block_queue.num_free_blocks:
273
+ raise ValueError(
274
+ f"Cannot get {num_blocks} free blocks from the pool")
275
+
276
+ # First allocate blocks.
277
+ ret: List[KVCacheBlock] = []
278
+ idx = 0
279
+ while idx < num_blocks:
280
+ curr_block = self.free_block_queue.popleft()
281
+ assert curr_block.ref_cnt == 0
282
+
283
+ # Evict blocks from the cache.
284
+ if self.enable_caching:
285
+ block_hash = curr_block.block_hash
286
+ if (block_hash is not None
287
+ and block_hash in self.cached_block_hash_to_block):
288
+ if len(self.cached_block_hash_to_block[block_hash]) == 1:
289
+ del self.cached_block_hash_to_block[block_hash]
290
+ else:
291
+ del self.cached_block_hash_to_block[block_hash][
292
+ curr_block.block_id]
293
+ curr_block.reset()
294
+
295
+ curr_block.ref_cnt = 1
296
+ ret.append(curr_block)
297
+ idx += 1
298
+
299
+ # Then assign token IDs to the allocated blocks.
300
+ if self.enable_caching:
301
+ assert token_ids is not None
302
+ token_id_idx = self._add_token_ids_to_blocks(
303
+ blocks=ret, token_ids=token_ids, parent_block=parent_block)
304
+ assert token_id_idx == len(token_ids)
305
+
306
+ return ret
307
+
308
+ def _cache_full_block(self,
309
+ block: KVCacheBlock,
310
+ parent_block: Optional[KVCacheBlock] = None) -> None:
311
+ """Cache a full block for prefix caching.
312
+
313
+ Args:
314
+ block: The block to cache.
315
+ parent_block: The parent block. None if this is the first block.
316
+ """
317
+ parent_block_hash = (parent_block.block_hash
318
+ if parent_block is not None else None)
319
+ assert len(block.token_ids) == self.block_size
320
+ block.token_ids = tuple(block.token_ids)
321
+ block_hash = hash_block_tokens(parent_block_hash, block.token_ids)
322
+ block.block_hash = block_hash
323
+ block.num_hashed_tokens = self.block_size + (
324
+ parent_block.num_hashed_tokens if parent_block is not None else 0)
325
+ self.cached_block_hash_to_block[block_hash][block.block_id] = block
326
+
327
+ def _get_cached_block(self,
328
+ block_hash: BlockHashType) -> Optional[KVCacheBlock]:
329
+ """Get a cached block by the block hash, or None if cache miss.
330
+ If there are duplicated blocks, we return the first block in the cache.
331
+
332
+ Args:
333
+ block_hash: The hash value of the block.
334
+
335
+ Returns:
336
+ The cached block if it exists, or None.
337
+ """
338
+ if block_hash in self.cached_block_hash_to_block:
339
+ first_block_id = list(
340
+ self.cached_block_hash_to_block[block_hash].keys())[0]
341
+ return self.cached_block_hash_to_block[block_hash][first_block_id]
342
+ return None
343
+
344
+ def _touch(self, blocks: List[KVCacheBlock]) -> None:
345
+ """Touch a block increases its reference count by 1, and may remove
346
+ the block from the free queue. This is used when a block is hit by
347
+ another request with the same prefix.
348
+
349
+ Args:
350
+ blocks: A list of blocks to touch.
351
+ """
352
+ for block in blocks:
353
+ # ref_cnt=0 means this block is in the free list (i.e. eviction
354
+ # candidate), so remove it.
355
+ if block.ref_cnt == 0:
356
+ self.free_block_queue.remove(block)
357
+ block.ref_cnt += 1
358
+
359
+ def _add_token_ids_to_blocks(
360
+ self,
361
+ blocks: List[KVCacheBlock],
362
+ token_ids: List[int],
363
+ parent_block: Optional[KVCacheBlock] = None) -> int:
364
+ """Add token IDs to a list of allocated blocks.
365
+ If a block becomes full after adding token IDs, cache it.
366
+ Return the token ID index that has not been added to the blocks
367
+ if the blocks are not enough to hold all the token IDs.
368
+
369
+ Args:
370
+ blocks: A list of blocks to add token IDs.
371
+ token_ids: A list of token IDs to add.
372
+ parent_block: The parent block. None if this is the
373
+ first block.
374
+
375
+ Returns:
376
+ The starting token ID index that has not been added to the blocks
377
+ due to insufficient given blocks.
378
+ """
379
+ token_id_start = 0
380
+ for curr_block in blocks:
381
+ # If all token IDs are added, then the rest of the blocks are
382
+ # preallocated blocks, so we only need to update the
383
+ # parent_block_id. FIXME
384
+ if token_id_start == len(token_ids):
385
+ continue
386
+
387
+ # Add token IDs to the empty slots in the block.
388
+ empty_slots = self.block_size - len(curr_block.token_ids)
389
+ token_id_end = min(token_id_start + empty_slots, len(token_ids))
390
+ curr_block.token_ids.extend(token_ids[token_id_start:token_id_end])
391
+ # Cache the block if it becomes full.
392
+ if len(curr_block.token_ids) == self.block_size:
393
+ self._cache_full_block(curr_block, parent_block)
394
+ parent_block = curr_block
395
+ token_id_start = token_id_end
396
+ return token_id_start
vllm/v1/core/kv_cache_utils.py
new
+194
@@ -0,0 +1,194 @@
1
+"""KV-Cache Utilities."""
2
+from dataclasses import dataclass, field
3
+from typing import List, Optional, Tuple, Union
4
+
5
+from vllm.logger import init_logger
6
+
7
+logger = init_logger(__name__)
8
+
9
+BlockHashType = Tuple[int, Tuple[int]]
10
+
11
+
12
+@dataclass
13
+class KVCacheBlock:
14
+ """KV-cache block metadata."""
15
+ # Block ID, ranging from 0 to num_gpu_blocks - 1.
16
+ block_id: int
17
+ # Reference count.
18
+ ref_cnt: int = 0
19
+ # Token IDs in the block. When the block is full, the type of token_ids
20
+ # should be Tuple[int] for fast matching.
21
+ token_ids: Union[List[int], Tuple[int]] = field(default_factory=list)
22
+ # The hash of the block composed of (block hash, tuple of token IDs).
23
+ # It is only available when the block is full.
24
+ block_hash: Optional[BlockHashType] = None
25
+ # The number of hashed tokens. More hashed tokens means the block
26
+ # is closer to the end of a prompt and more likely to be evicted.
27
+ num_hashed_tokens: int = 0
28
+
29
+ # Used to construct a doubly linked list for free blocks.
30
+ # These two attributes should only be manipulated by FreeKVCacheBlockQueue.
31
+ prev_free_block: Optional["KVCacheBlock"] = None
32
+ next_free_block: Optional["KVCacheBlock"] = None
33
+
34
+ def reset(self):
35
+ """Reset the block metadata."""
36
+ self.ref_cnt = 0
37
+ self.token_ids = []
38
+ self.block_hash = None
39
+ self.num_hashed_tokens = 0
40
+
41
+
42
+class FreeKVCacheBlockQueue:
43
+ """This class organizes a list of KVCacheBlock objects to a doubly linked
44
+ list of free blocks. We implement this class instead of using Python
45
+ builtin deque to support removing a block in the middle of the queue
46
+ in O(1) time. To close the performance gap to the builtin deque which is
47
+ implemented in C++, this class does not allocate any Python objects when
48
+ manipulating the linked list. Instead, this class manipulates the
49
+ prev_free_block and next_free_block attributes of the given blocks.
50
+
51
+ The queue is ordered by block ID in the beginning. When a block is allocated
52
+ and then freed, it will be appended back with the eviction order:
53
+ 1. The least recent used block is at the front (LRU).
54
+ 2. If two blocks have the same last accessed time (allocated by the
55
+ same sequence), the one with more hash tokens (the tail of a block
56
+ chain) is at the front.
57
+ Note that we maintain this order by reversing the block order when free
58
+ blocks of a request. This operation is outside of this class.
59
+
60
+ Args:
61
+ blocks: A list of KVCacheBlock objects.
62
+ """
63
+
64
+ def __init__(self, blocks: List[KVCacheBlock]) -> None:
65
+ self.num_free_blocks = len(blocks)
66
+
67
+ # Initialize the doubly linked list of free blocks.
68
+ self.free_list_head = blocks[0]
69
+ self.free_list_tail = blocks[-1]
70
+ for i in range(self.num_free_blocks):
71
+ if i > 0:
72
+ blocks[i].prev_free_block = blocks[i - 1]
73
+ if i < self.num_free_blocks - 1:
74
+ blocks[i].next_free_block = blocks[i + 1]
75
+
76
+ def popleft(self) -> KVCacheBlock:
77
+ """Pop the first free block and reduce num_free_blocks by 1.
78
+
79
+ Returns:
80
+ The first free block.
81
+ """
82
+ if not self.free_list_head:
83
+ raise ValueError("No free blocks available")
84
+
85
+ block = self.free_list_head
86
+ self.remove(block)
87
+ return block
88
+
89
+ def remove(self, block: KVCacheBlock) -> None:
90
+ """Remove a block in the free list and reduce num_free_blocks by 1.
91
+
92
+ Args:
93
+ block: The block to remove.
94
+ """
95
+ if block.prev_free_block is not None:
96
+ # Link the previous block to the next block.
97
+ block.prev_free_block.next_free_block = block.next_free_block
98
+ if block.next_free_block is not None:
99
+ # Link the next block to the previous block.
100
+ block.next_free_block.prev_free_block = block.prev_free_block
101
+
102
+ if block == self.free_list_head:
103
+ # Update the head if the block is the head.
104
+ self.free_list_head = block.next_free_block
105
+ if block == self.free_list_tail:
106
+ # Update the tail if the block is the tail.
107
+ self.free_list_tail = block.prev_free_block
108
+
109
+ # Remove the block from the linked list.
110
+ block.prev_free_block = block.next_free_block = None
111
+ self.num_free_blocks -= 1
112
+
113
+ def append(self, block: KVCacheBlock) -> None:
114
+ """Put a block back into the free list and increase
115
+ num_free_blocks by 1.
116
+
117
+ Args:
118
+ block: The block to append.
119
+ """
120
+ if self.free_list_tail is not None:
121
+ # Link the last block to the new block.
122
+ self.free_list_tail.next_free_block = block
123
+ block.prev_free_block = self.free_list_tail
124
+ self.free_list_tail = block
125
+ else:
126
+ # The free list is empty.
127
+ assert self.free_list_head is None
128
+ self.free_list_head = self.free_list_tail = block
129
+
130
+ block.next_free_block = None
131
+ self.num_free_blocks += 1
132
+
133
+ def get_all_free_blocks(self) -> List[KVCacheBlock]:
134
+ """Get all free blocks in the free list. Mainly used for testing.
135
+
136
+ Returns:
137
+ A list of free blocks.
138
+ """
139
+ ret = []
140
+ curr_block = self.free_list_head
141
+ while curr_block is not None:
142
+ ret.append(curr_block)
143
+ curr_block = curr_block.next_free_block
144
+ return ret
145
+
146
+
147
+def hash_block_tokens(parent_block_hash: Optional[int],
148
+ curr_block_token_ids: Tuple[int]) -> BlockHashType:
149
+ """Computes a hash value corresponding to the contents of a block and
150
+ the contents of the preceding block(s). The hash value is used for
151
+ prefix caching. We use LRU cache for this function to avoid recomputing
152
+ hash values for the same block contents.
153
+
154
+ TODO: Support arbitrary metadata so that we could support more
155
+ features such as LoRA adapter.
156
+
157
+ Args:
158
+ parent_block_hash: The hash of the parent block. None
159
+ if this is the first block.
160
+ curr_block_token_ids: A tuple of token ids in the current
161
+ block. The current block is assumed to be full.
162
+
163
+ Returns:
164
+ The hash value of the block and the token ids in the block.
165
+ The entire tuple is used as the hash key of the block.
166
+ """
167
+ return (hash(
168
+ (parent_block_hash, *curr_block_token_ids)), curr_block_token_ids)
169
+
170
+
171
+def hash_request_tokens(block_size: int,
172
+ token_ids: List[int]) -> List[BlockHashType]:
173
+ """Computes hash values of a chain of blocks given a sequence of
174
+ token IDs. The hash value is used for prefix caching.
175
+
176
+ Args:
177
+ block_size: The size of each block.
178
+ token_ids: A sequence of token ids in the request.
179
+
180
+ Returns:
181
+ The list of computed hash values.
182
+ """
183
+ ret = []
184
+ parent_block_hash = None
185
+ for start in range(0, len(token_ids), block_size):
186
+ end = start + block_size
187
+ block_token_ids = tuple(token_ids[start:end])
188
+ # Do not hash the block if it is not full.
189
+ if len(block_token_ids) < block_size:
190
+ break
191
+ block_hash = hash_block_tokens(parent_block_hash, block_token_ids)
192
+ ret.append(block_hash)
193
+ parent_block_hash = block_hash
194
+ return ret
vllm/v1/core/scheduler.py
+21
-11
@@ -34,7 +34,7 @@ class Scheduler:
34
block_size=self.cache_config.block_size,
35
num_gpu_blocks=num_gpu_blocks,
36
sliding_window=self.cache_config.sliding_window,
37
- enable_caching=True)
37
+ enable_caching=self.cache_config.enable_prefix_caching)
38
self.block_size = self.cache_config.block_size
39
40
# Scheduling constraints.
@@ -91,9 +91,9 @@ class Scheduler:
91
assert num_new_tokens > 0
92
93
while True:
94
- new_block_ids = self.kv_cache_manager.append_slots(
94
+ new_blocks = self.kv_cache_manager.append_slots(
95
request, num_new_tokens)
96
- if new_block_ids is None:
96
+ if new_blocks is None:
97
# The request cannot be scheduled.
98
# Preempt the lowest-priority request.
99
preempted_req = self.running.pop()
@@ -110,7 +110,9 @@ class Scheduler:
110
# The request can be scheduled.
111
scheduled_running_reqs.append(request)
112
113
- req_to_new_block_ids[request.request_id] = new_block_ids
113
+ req_to_new_block_ids[request.request_id] = [
114
+ b.block_id for b in new_blocks
115
+ ]
116
num_scheduled_tokens[request.request_id] = num_new_tokens
117
token_budget -= num_new_tokens
118
req_index += 1
@@ -126,22 +128,29 @@ class Scheduler:
128
129
request = self.waiting[0]
130
# Get already-cached tokens.
129
- computed_block_ids = self.kv_cache_manager.get_computed_blocks(
131
+ computed_blocks = self.kv_cache_manager.get_computed_blocks(
132
request)
133
# NOTE(woosuk): Since incomplete blocks are not eligible for
134
# sharing, `num_computed_tokens` is always a multiple of
135
# `block_size`.
134
- num_computed_tokens = len(computed_block_ids) * self.block_size
136
+ num_computed_tokens = len(computed_blocks) * self.block_size
137
# Number of tokens to be scheduled.
138
# We use `request.num_tokens` instead of
139
# `request.num_prompt_tokens` to consider the resumed requests,
140
# which have output tokens.
141
num_new_tokens = request.num_tokens - num_computed_tokens
142
+ if num_new_tokens == 0:
143
+ # The happens when prompt length is divisible by the block
144
+ # size and all blocks are cached. Now we force to recompute
145
+ # the last token.
146
+ num_computed_tokens -= 1
147
+ num_new_tokens = 1
148
+ computed_blocks.pop()
149
num_new_tokens = min(num_new_tokens, token_budget)
150
assert num_new_tokens > 0
142
- new_block_ids = self.kv_cache_manager.allocate_slots(
143
- request, num_new_tokens, computed_block_ids)
144
- if new_block_ids is None:
151
+ new_blocks = self.kv_cache_manager.allocate_slots(
152
+ request, num_new_tokens, computed_blocks)
153
+ if new_blocks is None:
154
# The request cannot be scheduled.
155
break
156
request.num_computed_tokens = num_computed_tokens
@@ -156,8 +165,9 @@ class Scheduler:
165
raise RuntimeError(
166
f"Invalid request status: {request.status}")
167
159
- req_to_new_block_ids[request.request_id] = (
160
- computed_block_ids + new_block_ids)
168
+ req_to_new_block_ids[request.request_id] = [
169
+ b.block_id for b in computed_blocks + new_blocks
170
+ ]
171
num_scheduled_tokens[request.request_id] = num_new_tokens
172
token_budget -= num_new_tokens
173
request.status = RequestStatus.RUNNING
vllm/v1/engine/llm_engine.py
+1
@@ -65,6 +65,7 @@ class LLMEngine:
65
elif usage_context == UsageContext.OPENAI_API_SERVER:
66
scheduler_config.max_num_seqs = 1024
67
scheduler_config.max_num_batched_tokens = 2048
68
+ cache_config.enable_prefix_caching = True
69
70
logger.info(
71
"Initializing an LLM engine (v%s) with config: "