A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Add kv cache utils tests. (#11513)
Signed-off-by: xcnick <xcnick0412@gmail.com>
xcnick committed
Jan 4, 2025 at 14:49 UTC
d91457d529c2df5d66bdfd939b90b7c75a9729b8
1 file changed
+241
tests/v1/core/test_kv_cache_utils.py
new
+241
@@ -0,0 +1,241 @@
1
+import pytest
2
+
3
+from vllm.inputs import token_inputs
4
+from vllm.sampling_params import SamplingParams
5
+from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
6
+ KVCacheBlock,
7
+ generate_block_hash_extra_keys,
8
+ hash_block_tokens,
9
+ hash_request_tokens)
10
+from vllm.v1.request import Request
11
+
12
+
13
+def make_request(request_id,
14
+ prompt_token_ids,
15
+ mm_positions=None,
16
+ mm_hashes=None):
17
+ return Request(
18
+ request_id=request_id,
19
+ inputs=token_inputs(
20
+ prompt_token_ids=prompt_token_ids,
21
+ multi_modal_placeholders={"image": mm_positions}
22
+ if mm_positions else None,
23
+ multi_modal_hashes=mm_hashes,
24
+ ),
25
+ sampling_params=SamplingParams(max_tokens=17),
26
+ eos_token_id=100,
27
+ arrival_time=0,
28
+ lora_request=None,
29
+ )
30
+
31
+
32
+def test_kv_cache_block():
33
+ # Test KVCacheBlock initialization
34
+ block = KVCacheBlock(block_id=0)
35
+ assert block.block_id == 0
36
+ assert block.ref_cnt == 0
37
+ assert block.block_hash is None
38
+
39
+ # Test reference count manipulation
40
+ block.incr_ref()
41
+ assert block.ref_cnt == 1
42
+ block.decr_ref()
43
+ assert block.ref_cnt == 0
44
+
45
+ # Test block hash setting and resetting
46
+ block_hash = BlockHashType(hash_value=123, token_ids=(1, 2, 3))
47
+ block.block_hash = block_hash
48
+ assert block.block_hash == block_hash
49
+
50
+ block.reset_hash()
51
+ assert block.block_hash is None
52
+
53
+
54
+def test_free_kv_cache_block_queue_initialization():
55
+ # Test with a single block
56
+ block = KVCacheBlock(block_id=0)
57
+ queue = FreeKVCacheBlockQueue([block])
58
+ assert queue.num_free_blocks == 1
59
+ assert queue.free_list_head == block
60
+ assert queue.free_list_tail == block
61
+
62
+
63
+def test_free_kv_cache_block_queue_operations():
64
+ # Create a list of KVCacheBlock objects
65
+ blocks = [KVCacheBlock(block_id=i) for i in range(5)]
66
+
67
+ # Create a FreeKVCacheBlockQueue with these blocks
68
+ queue = FreeKVCacheBlockQueue(blocks)
69
+
70
+ # Check initial state
71
+ assert queue.num_free_blocks == 5
72
+ assert queue.free_list_head == blocks[0]
73
+ assert queue.free_list_tail == blocks[4]
74
+
75
+ # Pop the first block
76
+ block1 = queue.popleft()
77
+ assert block1 == blocks[0]
78
+ assert queue.num_free_blocks == 4
79
+ assert queue.free_list_head == blocks[1]
80
+ assert queue.free_list_tail == blocks[4]
81
+
82
+ # Remove a block from the middle
83
+ block_to_remove = blocks[2]
84
+ queue.remove(block_to_remove)
85
+ assert queue.num_free_blocks == 3
86
+ assert blocks[1].next_free_block == blocks[3]
87
+ assert blocks[3].prev_free_block == blocks[1]
88
+
89
+ # Append a block back
90
+ queue.append(block_to_remove)
91
+ assert queue.num_free_blocks == 4
92
+ assert queue.free_list_tail == block_to_remove
93
+ assert block_to_remove.prev_free_block == blocks[4]
94
+ assert block_to_remove.next_free_block is None
95
+
96
+ # Pop blocks until empty
97
+ for _ in range(4):
98
+ queue.popleft()
99
+ assert queue.num_free_blocks == 0
100
+ assert queue.free_list_head is None
101
+ assert queue.free_list_tail is None
102
+
103
+ # Attempt to pop from an empty queue
104
+ with pytest.raises(ValueError) as e:
105
+ queue.popleft()
106
+ assert str(e.value) == "No free blocks available"
107
+
108
+
109
+def test_free_kv_cache_block_queue_get_all_free_blocks():
110
+ # Create a list of KVCacheBlock objects
111
+ blocks = [KVCacheBlock(block_id=i) for i in range(5)]
112
+
113
+ # Create a FreeKVCacheBlockQueue with these blocks
114
+ queue = FreeKVCacheBlockQueue(blocks)
115
+
116
+ # Check all blocks are correctly retrieved
117
+ assert queue.get_all_free_blocks() == blocks
118
+
119
+ # Pop a block and check again
120
+ queue.popleft()
121
+ assert queue.get_all_free_blocks() == blocks[1:]
122
+
123
+ # Remove a block and check again
124
+ block_to_remove = blocks[2]
125
+ queue.remove(block_to_remove)
126
+ assert queue.get_all_free_blocks() == blocks[1:2] + blocks[3:]
127
+
128
+ # Append a block back and check again
129
+ queue.append(block_to_remove)
130
+ assert queue.get_all_free_blocks() == \
131
+ blocks[1:2] + blocks[3:] + [block_to_remove]
132
+
133
+
134
+def test_generate_block_hash_extra_keys():
135
+ request = make_request(
136
+ request_id=0,
137
+ prompt_token_ids=[_ for _ in range(20)],
138
+ mm_positions=[{
139
+ "offset": 0,
140
+ "length": 5
141
+ }, {
142
+ "offset": 10,
143
+ "length": 5
144
+ }],
145
+ mm_hashes=["hash1", "hash2"],
146
+ )
147
+
148
+ # Test with no extra keys
149
+ extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
150
+ assert extra_keys == (("hash1", 0), )
151
+ assert next_mm_idx == 1
152
+
153
+ # Test with partial overlap
154
+ extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
155
+ assert extra_keys == (("hash1", 3), )
156
+ assert next_mm_idx == 1
157
+
158
+ # Test with no overlap
159
+ extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 6, 10, 0)
160
+ assert extra_keys == ()
161
+ assert next_mm_idx == 1
162
+
163
+ # Test with multiple extra keys
164
+ extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
165
+ assert extra_keys == (("hash1", 0), ("hash2", 0))
166
+ assert next_mm_idx == 2
167
+
168
+
169
+def test_generate_block_hash_extra_keys_no_mm_inputs():
170
+ request = make_request(
171
+ request_id=0,
172
+ prompt_token_ids=[_ for _ in range(6)],
173
+ mm_positions=None,
174
+ mm_hashes=None,
175
+ )
176
+
177
+ extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
178
+ assert extra_keys is None
179
+ assert next_mm_idx == 0
180
+
181
+
182
+def test_hash_block_tokens():
183
+ parent_block_hash = 123
184
+ curr_block_token_ids = (1, 2, 3)
185
+ extra_keys = ("key1", "key2")
186
+
187
+ block_hash = hash_block_tokens(parent_block_hash, curr_block_token_ids,
188
+ extra_keys)
189
+ assert isinstance(block_hash, BlockHashType)
190
+ assert block_hash.hash_value == hash(
191
+ (parent_block_hash, *curr_block_token_ids))
192
+ assert block_hash.token_ids == curr_block_token_ids
193
+ assert block_hash.extra_keys == extra_keys
194
+
195
+
196
+def test_hash_request_tokens():
197
+ request = make_request(
198
+ request_id=0,
199
+ prompt_token_ids=[_ for _ in range(6)],
200
+ mm_positions=[{
201
+ "offset": 0,
202
+ "length": 3
203
+ }, {
204
+ "offset": 3,
205
+ "length": 3
206
+ }],
207
+ mm_hashes=["hash1", "hash2"],
208
+ )
209
+
210
+ block_size = 3
211
+ block_hashes = hash_request_tokens(block_size, request)
212
+
213
+ assert len(block_hashes) == 2
214
+ assert isinstance(block_hashes[0], BlockHashType)
215
+ assert isinstance(block_hashes[1], BlockHashType)
216
+
217
+ # Check the first block
218
+ assert block_hashes[0].token_ids == (0, 1, 2)
219
+ assert block_hashes[0].extra_keys == (("hash1", 0), )
220
+
221
+ # Check the second block
222
+ assert block_hashes[1].token_ids == (3, 4, 5)
223
+ assert block_hashes[1].extra_keys == (("hash2", 0), )
224
+
225
+
226
+def test_hash_request_tokens_no_mm_inputs():
227
+ request = make_request(
228
+ request_id=0,
229
+ prompt_token_ids=[_ for _ in range(6)],
230
+ mm_positions=None,
231
+ mm_hashes=None,
232
+ )
233
+
234
+ block_size = 3
235
+ block_hashes = hash_request_tokens(block_size, request)
236
+
237
+ assert len(block_hashes) == 2
238
+ assert block_hashes[0].token_ids == (0, 1, 2)
239
+ assert block_hashes[0].extra_keys is None
240
+ assert block_hashes[1].token_ids == (3, 4, 5)
241
+ assert block_hashes[1].extra_keys is None