A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Prefix caching for vision language models (#11187)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
Cody Yu committed
Dec 17, 2024 at 16:37 UTC
bf8717ebaea8d74279df84fbe127ad22cf62e219
14 files changed
+342
-98
tests/v1/core/test_prefix_caching.py
+86
-2
@@ -2,16 +2,23 @@
2
import pytest
3
4
from vllm.inputs import token_inputs
5
+from vllm.multimodal.inputs import PlaceholderRange
6
from vllm.sampling_params import SamplingParams
7
from vllm.utils import cdiv
8
from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
9
from vllm.v1.core.kv_cache_utils import KVCacheBlock, hash_block_tokens
10
11
11
-def make_request(request_id, prompt_token_ids):
12
+def make_request(request_id,
13
+ prompt_token_ids,
14
+ mm_positions=None,
15
+ mm_hashes=None):
16
return Request(
17
request_id=request_id,
14
- inputs=token_inputs(prompt_token_ids=prompt_token_ids),
18
+ inputs=token_inputs(prompt_token_ids=prompt_token_ids,
19
+ multi_modal_placeholders={"image": mm_positions}
20
+ if mm_positions else None,
21
+ multi_modal_hashes=mm_hashes),
22
sampling_params=SamplingParams(max_tokens=17),
23
eos_token_id=100,
24
arrival_time=0,
@@ -38,6 +45,7 @@ def test_prefill():
45
all_token_ids = common_token_ids + unique_token_ids
46
req0 = make_request("0", all_token_ids)
47
computed_blocks = manager.get_computed_blocks(req0)
48
+ assert len(req0.kv_block_hashes) == 3
49
assert not computed_blocks
50
blocks = manager.allocate_slots(req0, 55, computed_blocks)
51
assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
@@ -61,6 +69,7 @@ def test_prefill():
69
unique_token_ids = [3] * 5
70
req1 = make_request("1", common_token_ids + unique_token_ids)
71
computed_blocks = manager.get_computed_blocks(req1)
72
+ assert len(req1.kv_block_hashes) == 3
73
assert [b.block_id for b in computed_blocks] == [0, 1, 2]
74
num_new_tokens = 53 - 3 * 16
75
blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
@@ -90,6 +99,7 @@ def test_prefill():
99
unique_token_ids = [3] * 6
100
req2 = make_request("2", common_token_ids + unique_token_ids)
101
computed_block = manager.get_computed_blocks(req2)
102
+ assert len(req2.kv_block_hashes) == 3
103
assert [b.block_id for b in computed_block] == [0, 1, 2]
104
num_new_tokens = 53 - 3 * 16
105
blocks = manager.allocate_slots(req2, num_new_tokens, computed_blocks)
@@ -416,3 +426,77 @@ def test_cache_blocks():
426
)
427
assert len(manager.cached_block_hash_to_block) == 3
428
assert blocks[0].block_hash is not None
429
+
430
+
431
+def test_mm_prefix_caching():
432
+ """
433
+ This tests that the multi-modal prefix caching is correct.
434
+ """
435
+ manager = KVCacheManager(
436
+ block_size=16,
437
+ num_gpu_blocks=10,
438
+ max_model_len=8192,
439
+ sliding_window=None,
440
+ enable_caching=True,
441
+ num_preallocate_tokens=16,
442
+ )
443
+
444
+ # Common prompt tokens (T is text tokens and P is image placeholder tokens)
445
+ # [T,...,T, P0,...,P0], [P0,...,P0,T,...,T,P1,...,P1], [P1,...,P1]
446
+ common_token_ids = list(range(10)) + [-1] * 6
447
+ common_token_ids += [-1] * 4 + list(range(10, 20)) + [-1] * 2
448
+ common_token_ids += [-1] * 16
449
+
450
+ common_mm_positions = [
451
+ PlaceholderRange(offset=11, length=10),
452
+ PlaceholderRange(offset=30, length=18),
453
+ ]
454
+ common_mm_hashes = ["aaa", "bbb"]
455
+
456
+ # A unique image plus some text tokens.
457
+ unique_token_ids = [-1] * 7 + [100] * 4
458
+ all_token_ids = common_token_ids + unique_token_ids
459
+ mm_positions = common_mm_positions + [
460
+ PlaceholderRange(offset=48, length=7)
461
+ ]
462
+ mm_hashes = common_mm_hashes + ["ccc"]
463
+ req0 = make_request("0",
464
+ all_token_ids,
465
+ mm_positions=mm_positions,
466
+ mm_hashes=mm_hashes)
467
+ computed_blocks = manager.get_computed_blocks(req0)
468
+
469
+ # Completed block should have hashes with extra keys.
470
+ assert not computed_blocks
471
+ assert len(req0.kv_block_hashes) == 3
472
+ assert req0.kv_block_hashes[0].extra_keys == (("aaa", 0), )
473
+ assert req0.kv_block_hashes[1].extra_keys == (("aaa", 5), ("bbb", 0))
474
+ assert req0.kv_block_hashes[2].extra_keys == (("bbb", 2), )
475
+
476
+ blocks = manager.allocate_slots(req0, 59, computed_blocks)
477
+ assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
478
+ req0.num_computed_tokens = 59
479
+
480
+ # Append slots without allocating a new block.
481
+ for _ in range(5):
482
+ req0.append_output_token_ids(8)
483
+ new_blocks = manager.append_slots(req0, 5)
484
+ assert new_blocks is not None and len(new_blocks) == 0
485
+
486
+ # The just completed block should have hashes with extra keys.
487
+ assert len(req0.kv_block_hashes) == 4
488
+ assert req0.kv_block_hashes[3].extra_keys == (("ccc", 0), )
489
+
490
+ # Cache hit.
491
+ unique_token_ids = [-1] * 7 + [200] * 5
492
+ all_token_ids = common_token_ids + unique_token_ids
493
+ mm_positions = common_mm_positions + [
494
+ PlaceholderRange(offset=48, length=7)
495
+ ]
496
+ mm_hashes = common_mm_hashes + ["ccc"]
497
+ req1 = make_request("1",
498
+ all_token_ids,
499
+ mm_positions=mm_positions,
500
+ mm_hashes=mm_hashes)
501
+ computed_blocks = manager.get_computed_blocks(req1)
502
+ assert len(computed_blocks) == 3
tests/v1/engine/test_engine_args.py
-15
@@ -31,14 +31,6 @@ def test_prefix_caching_from_cli():
31
assert engine_args.enable_prefix_caching
32
33
34
-def test_defaults():
35
- engine_args = EngineArgs(model="facebook/opt-125m")
36
-
37
- # Assert V1 defaults
38
- assert (engine_args.enable_prefix_caching
39
- ), "V1 turns on prefix caching by default"
40
-
41
-
34
def test_defaults_with_usage_context():
35
engine_args = EngineArgs(model="facebook/opt-125m")
36
vllm_config: VllmConfig = engine_args.create_engine_config(
@@ -52,10 +44,3 @@ def test_defaults_with_usage_context():
44
UsageContext.OPENAI_API_SERVER)
45
assert vllm_config.scheduler_config.max_num_seqs == 1024
46
assert vllm_config.scheduler_config.max_num_batched_tokens == 2048
55
-
56
-
57
-def test_prefix_cache_disabled_with_multimodel():
58
- engine_args = EngineArgs(model="llava-hf/llava-1.5-7b-hf")
59
-
60
- vllm_config = engine_args.create_engine_config(UsageContext.LLM_CLASS)
61
- assert not vllm_config.cache_config.enable_prefix_caching
vllm/engine/arg_utils.py
+14
-13
@@ -205,6 +205,7 @@ class EngineArgs:
205
# by user.
206
if self.enable_prefix_caching is None:
207
self.enable_prefix_caching = bool(envs.VLLM_USE_V1)
208
+
209
# Override max_num_seqs if it's not set by user.
210
if self.max_num_seqs is None:
211
self.max_num_seqs = 256 if not envs.VLLM_USE_V1 else 1024
@@ -1026,11 +1027,11 @@ class EngineArgs:
1027
device_config = DeviceConfig(device=self.device)
1028
model_config = self.create_model_config()
1029
1029
- if model_config.is_multimodal_model:
1030
- if self.enable_prefix_caching:
1031
- logger.warning(
1032
- "--enable-prefix-caching is currently not "
1033
- "supported for multimodal models and has been disabled.")
1030
+ if (model_config.is_multimodal_model and not envs.VLLM_USE_V1
1031
+ and self.enable_prefix_caching):
1032
+ logger.warning("--enable-prefix-caching is currently not "
1033
+ "supported for multimodal models in v0 and "
1034
+ "has been disabled.")
1035
self.enable_prefix_caching = False
1036
1037
cache_config = CacheConfig(
@@ -1249,11 +1250,14 @@ class EngineArgs:
1250
# When no user override, set the default values based on the usage
1251
# context.
1252
# TODO(woosuk): Tune the default values for different hardware.
1252
- if self.max_num_batched_tokens is None:
1253
- if usage_context == UsageContext.LLM_CLASS:
1254
- self.max_num_batched_tokens = 8192
1255
- elif usage_context == UsageContext.OPENAI_API_SERVER:
1256
- self.max_num_batched_tokens = 2048
1253
+ default_max_num_batched_tokens = {
1254
+ UsageContext.LLM_CLASS: 8192,
1255
+ UsageContext.OPENAI_API_SERVER: 2048,
1256
+ }
1257
+ if (self.max_num_batched_tokens is None
1258
+ and usage_context in default_max_num_batched_tokens):
1259
+ self.max_num_batched_tokens = default_max_num_batched_tokens[
1260
+ usage_context]
1261
logger.warning(
1262
"Setting max_num_batched_tokens to %d for %s usage context.",
1263
self.max_num_batched_tokens, usage_context.value)
@@ -1263,9 +1267,6 @@ class EngineArgs:
1267
Override the EngineConfig's configs based on the usage context for V1.
1268
"""
1269
assert envs.VLLM_USE_V1, "V1 is not enabled"
1266
- if engine_config.model_config.is_multimodal_model:
1267
- # TODO (ywang96): Enable APC by default when VLM supports it.
1268
- assert not engine_config.cache_config.enable_prefix_caching
1270
1271
1272
@dataclass
vllm/inputs/data.py
+20
@@ -162,6 +162,11 @@ class TokenInputs(TypedDict):
162
Placeholder ranges for the multi-modal data.
163
"""
164
165
+ multi_modal_hashes: NotRequired[List[str]]
166
+ """
167
+ The hashes of the multi-modal data.
168
+ """
169
+
170
mm_processor_kwargs: NotRequired[Dict[str, Any]]
171
"""
172
Optional multi-modal processor kwargs to be forwarded to the
@@ -177,6 +182,7 @@ def token_inputs(
182
prompt: Optional[str] = None,
183
multi_modal_data: Optional["MultiModalDataDict"] = None,
184
multi_modal_inputs: Optional["MultiModalKwargs"] = None,
185
+ multi_modal_hashes: Optional[List[str]] = None,
186
multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
187
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
188
) -> TokenInputs:
@@ -191,6 +197,8 @@ def token_inputs(
197
inputs["multi_modal_data"] = multi_modal_data
198
if multi_modal_inputs is not None:
199
inputs["multi_modal_inputs"] = multi_modal_inputs
200
+ if multi_modal_hashes is not None:
201
+ inputs["multi_modal_hashes"] = multi_modal_hashes
202
if multi_modal_placeholders is not None:
203
inputs["multi_modal_placeholders"] = multi_modal_placeholders
204
if mm_processor_kwargs is not None:
@@ -295,6 +303,18 @@ class SingletonInputsAdapter:
303
304
assert_never(inputs)
305
306
+ @cached_property
307
+ def multi_modal_hashes(self) -> List[str]:
308
+ inputs = self.inputs
309
+
310
+ if inputs["type"] == "token":
311
+ return inputs.get("multi_modal_hashes", [])
312
+
313
+ if inputs["type"] == "multimodal":
314
+ return inputs.get("mm_hashes", [])
315
+
316
+ assert_never(inputs)
317
+
318
@cached_property
319
def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
320
inputs = self.inputs
vllm/multimodal/inputs.py
+3
@@ -215,6 +215,9 @@ class MultiModalInputsV2(TypedDict):
215
mm_kwargs: MultiModalKwargs
216
"""Keyword arguments to be directly passed to the model after batching."""
217
218
+ mm_hashes: NotRequired[List[str]]
219
+ """The hashes of the multi-modal data."""
220
+
221
mm_placeholders: MultiModalPlaceholderDict
222
"""
223
For each modality, information about the placeholder tokens in
vllm/v1/core/kv_cache_manager.py
+50
-24
@@ -4,7 +4,9 @@ from typing import Dict, Iterable, List, Optional
4
from vllm.logger import init_logger
5
from vllm.utils import cdiv
6
from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
7
- KVCacheBlock, hash_block_tokens,
7
+ KVCacheBlock,
8
+ generate_block_hash_extra_keys,
9
+ hash_block_tokens,
10
hash_request_tokens)
11
from vllm.v1.request import Request
12
@@ -83,10 +85,12 @@ class KVCacheManager:
85
86
computed_blocks = []
87
86
- # TODO(rickyx): potentially we could cache this so we don't have to
87
- # recompute it every time.
88
- block_hashes = hash_request_tokens(self.block_size,
89
- request.all_token_ids)
88
+ # The block hashes for the request may already be computed
89
+ # if the request was preempted and resumed.
90
+ if not request.kv_block_hashes:
91
+ request.set_kv_block_hashes(
92
+ hash_request_tokens(self.block_size, request))
93
+ block_hashes = request.kv_block_hashes
94
95
for block_hash in block_hashes:
96
# block_hashes is a chain of block hashes. If a block hash is not
@@ -242,14 +246,16 @@ class KVCacheManager:
246
num_computed_tokens = len(computed_blocks) * self.block_size
247
num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
248
245
- self._cache_full_blocks(
246
- request=request,
247
- blk_start_idx=len(computed_blocks),
248
- # The new full blocks are the full blocks that are not computed.
249
- full_blocks=self.req_to_blocks[request.request_id]
250
- [len(computed_blocks):num_full_blocks],
251
- prev_block=computed_blocks[-1] if computed_blocks else None,
252
- )
249
+ new_full_blocks = self.req_to_blocks[
250
+ request.request_id][len(computed_blocks):num_full_blocks]
251
+ if new_full_blocks:
252
+ self._cache_full_blocks(
253
+ request=request,
254
+ blk_start_idx=len(computed_blocks),
255
+ # The new full blocks are the full blocks that are not computed.
256
+ full_blocks=new_full_blocks,
257
+ prev_block=computed_blocks[-1] if computed_blocks else None,
258
+ )
259
260
return new_blocks
261
@@ -376,6 +382,8 @@ class KVCacheManager:
382
full_blocks: The list of blocks to update hash metadata.
383
prev_block: The previous block in the chain.
384
"""
385
+ num_cached_block_hashes = len(request.kv_block_hashes)
386
+
387
# Update the new blocks with the block hashes through the chain.
388
prev_block_hash_value = None
389
if prev_block is not None:
@@ -387,17 +395,35 @@ class KVCacheManager:
395
for i, blk in enumerate(full_blocks):
396
blk_idx = blk_start_idx + i
397
390
- block_tokens = request.all_token_ids[blk_idx *
391
- self.block_size:(blk_idx +
392
- 1) *
393
- self.block_size]
394
- assert len(block_tokens) == self.block_size, (
395
- f"Expected {self.block_size} tokens, got {len(block_tokens)} "
396
- f"at {blk_idx}th block for request "
397
- f"{request.request_id}({request})")
398
-
399
- # Compute the hash of the current block.
400
- block_hash = hash_block_tokens(prev_block_hash_value, block_tokens)
398
+ if blk_idx < num_cached_block_hashes:
399
+ # The block hash may already be computed in
400
+ # "get_computed_blocks" if the tokens are not generated by
401
+ # this request (either the prompt tokens or the previously
402
+ # generated tokens with preemption). In this case we simply
403
+ # reuse the block hash.
404
+ block_hash = request.kv_block_hashes[blk_idx]
405
+ else:
406
+ # Otherwise compute the block hash and cache it in the request
407
+ # in case it will be preempted in the future.
408
+ start_token_idx = blk_idx * self.block_size
409
+ end_token_idx = (blk_idx + 1) * self.block_size
410
+ block_tokens = request.all_token_ids[
411
+ start_token_idx:end_token_idx]
412
+ assert len(block_tokens) == self.block_size, (
413
+ f"Expected {self.block_size} tokens, got "
414
+ f"{len(block_tokens)} at {blk_idx}th block for request "
415
+ f"{request.request_id}({request})")
416
+
417
+ # Generate extra keys for multi-modal inputs. Note that since
418
+ # we reach to this branch only when the block is completed with
419
+ # generated tokens, we only need to consider the last mm input.
420
+ extra_keys, _ = generate_block_hash_extra_keys(
421
+ request, start_token_idx, end_token_idx, -1)
422
+
423
+ # Compute the hash of the current block.
424
+ block_hash = hash_block_tokens(prev_block_hash_value,
425
+ block_tokens, extra_keys)
426
+ request.append_kv_block_hashes(block_hash)
427
428
# Update and added the full block to the cache.
429
blk.block_hash = block_hash
vllm/v1/core/kv_cache_utils.py
+105
-10
@@ -1,20 +1,25 @@
1
"""KV-Cache Utilities."""
2
from collections.abc import Sequence
3
from dataclasses import dataclass
4
-from typing import List, NamedTuple, Optional, Tuple
4
+from typing import Any, List, NamedTuple, Optional, Tuple
5
6
from vllm.logger import init_logger
7
+from vllm.v1.request import Request
8
9
logger = init_logger(__name__)
10
11
12
class BlockHashType(NamedTuple):
12
- """Hash value of a block and the token IDs in the block.
13
- The reason we keep a tuple of token IDs is to make sure no hash
14
- collision happens when the hash value is the same.
13
+ """Hash value of a block (int), the token IDs in the block, and extra keys.
14
+ The reason we keep a tuple of token IDs and extra keys is to make sure
15
+ no hash collision happens when the hash value is the same.
16
"""
17
+ # Hash value of the block in an integer.
18
hash_value: int
19
+ # Token IDs in the block.
20
token_ids: Tuple[int, ...]
21
+ # Extra keys for the block.
22
+ extra_keys: Optional[Any] = None
23
24
25
@dataclass
@@ -159,8 +164,80 @@ class FreeKVCacheBlockQueue:
164
return ret
165
166
162
-def hash_block_tokens(parent_block_hash: Optional[int],
163
- curr_block_token_ids: Sequence[int]) -> BlockHashType:
167
+def generate_block_hash_extra_keys(
168
+ request: Request, start_token_idx: int, end_token_idx: int,
169
+ start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:
170
+ """Generate extra keys for the block hash. The extra keys can come from
171
+ the multi-modal inputs and request specific metadata (e.g., LoRA ID).
172
+ For multi-modal inputs, the extra keys are (mm_hash, start_offset) that
173
+ indicate a mm input contained in the block and its starting offset in
174
+ the block tokens.
175
+
176
+ Args:
177
+ request: The request object.
178
+ start_token_idx: The start token index of the block.
179
+ end_token_idx: The end token index of the block.
180
+ start_mm_idx: The start multi-modal index of the block.
181
+
182
+ Returns:
183
+ A tuple of extra keys and the next multi-modal index.
184
+ """
185
+
186
+ mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
187
+ if not mm_positions:
188
+ return None, start_mm_idx
189
+
190
+ if mm_positions and len(mm_positions) != len(mm_hashes):
191
+ raise ValueError(
192
+ "The number of multi-modal positions and hashes must match. This "
193
+ "is likely because you do not enable MM preprocessor hashing. "
194
+ "Please set mm_cache_preprocessor=True.")
195
+
196
+ # Note that we assume mm_positions is sorted by offset.
197
+ # We do not need to check all mm inputs if the start token index is out of
198
+ # range. This usually happens in the late prefill phase and decoding phase.
199
+ if mm_positions[-1]["offset"] + mm_positions[-1][
200
+ "length"] < start_token_idx:
201
+ return None, start_mm_idx
202
+
203
+ # Support start_mm_idx == -1 to indicate the last mm input.
204
+ if start_mm_idx < 0:
205
+ assert -start_mm_idx <= len(mm_positions)
206
+ start_mm_idx = len(mm_positions) + start_mm_idx
207
+
208
+ extra_keys = []
209
+ curr_mm_idx = start_mm_idx
210
+ while mm_positions and curr_mm_idx < len(mm_positions):
211
+ assert mm_hashes[curr_mm_idx] is not None
212
+ offset = mm_positions[curr_mm_idx]["offset"]
213
+ length = mm_positions[curr_mm_idx]["length"]
214
+ if end_token_idx > offset:
215
+ if start_token_idx > offset + length:
216
+ # This block has passed the current mm input.
217
+ curr_mm_idx += 1
218
+ continue
219
+
220
+ # The block contains the current mm input.
221
+ mm_start = max(0, start_token_idx - offset)
222
+ extra_keys.append((mm_hashes[curr_mm_idx], mm_start))
223
+ if end_token_idx >= offset + length:
224
+ # If this block contains the end of the current mm input,
225
+ # move to the next mm input as this block may also contain
226
+ # the next mm input.
227
+ curr_mm_idx += 1
228
+ else:
229
+ # Otherwise this block is done with mm inputs.
230
+ break
231
+ else:
232
+ # This block has not reached the current mm input.
233
+ break
234
+ return tuple(extra_keys), curr_mm_idx
235
+
236
+
237
+def hash_block_tokens(
238
+ parent_block_hash: Optional[int],
239
+ curr_block_token_ids: Sequence[int],
240
+ extra_keys: Optional[Tuple[Any, ...]] = None) -> BlockHashType:
241
"""Computes a hash value corresponding to the contents of a block and
242
the contents of the preceding block(s). The hash value is used for
243
prefix caching. We use LRU cache for this function to avoid recomputing
@@ -174,27 +251,39 @@ def hash_block_tokens(parent_block_hash: Optional[int],
251
if this is the first block.
252
curr_block_token_ids: A list of token ids in the current
253
block. The current block is assumed to be full.
254
+ extra_keys: Extra keys for the block.
255
256
Returns:
257
The hash value of the block and the token ids in the block.
258
The entire tuple is used as the hash key of the block.
259
"""
260
return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
183
- tuple(curr_block_token_ids))
261
+ tuple(curr_block_token_ids), extra_keys)
262
263
264
def hash_request_tokens(block_size: int,
187
- token_ids: Sequence[int]) -> List[BlockHashType]:
265
+ request: Request) -> List[BlockHashType]:
266
"""Computes hash values of a chain of blocks given a sequence of
267
token IDs. The hash value is used for prefix caching.
268
269
Args:
270
block_size: The size of each block.
193
- token_ids: A sequence of token ids in the request.
271
+ request: The request object.
272
273
Returns:
274
The list of computed hash values.
275
"""
276
+ token_ids = request.all_token_ids
277
+ mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
278
+ if mm_positions and len(mm_positions) != len(mm_hashes):
279
+ raise ValueError(
280
+ "The number of multi-modal positions and hashes must match.")
281
+
282
+ # TODO: Extend this to support other features such as LoRA.
283
+ need_extra_keys = bool(mm_positions)
284
+ extra_keys = None
285
+ curr_mm_idx = 0
286
+
287
ret = []
288
parent_block_hash_value = None
289
for start in range(0, len(token_ids), block_size):
@@ -203,8 +292,14 @@ def hash_request_tokens(block_size: int,
292
# Do not hash the block if it is not full.
293
if len(block_token_ids) < block_size:
294
break
295
+
296
+ # Add extra keys if the block is a multi-modal block.
297
+ if need_extra_keys:
298
+ extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
299
+ request, start, end, curr_mm_idx)
300
+
301
block_hash = hash_block_tokens(parent_block_hash_value,
207
- block_token_ids)
302
+ block_token_ids, extra_keys)
303
ret.append(block_hash)
304
parent_block_hash_value = block_hash.hash_value
305
return ret
vllm/v1/core/scheduler.py
+2
@@ -516,6 +516,7 @@ class NewRequestData:
516
prompt_token_ids: List[int]
517
prompt: Optional[str]
518
mm_inputs: List["MultiModalKwargs"]
519
+ mm_hashes: List[str]
520
mm_positions: List["PlaceholderRange"]
521
sampling_params: SamplingParams
522
block_ids: List[int]
@@ -533,6 +534,7 @@ class NewRequestData:
534
prompt_token_ids=request.prompt_token_ids,
535
prompt=request.prompt,
536
mm_inputs=request.mm_inputs,
537
+ mm_hashes=request.mm_hashes,
538
mm_positions=request.mm_positions,
539
sampling_params=request.sampling_params,
540
block_ids=block_ids,
vllm/v1/engine/async_llm.py
+7
-3
@@ -60,9 +60,13 @@ class AsyncLLM(EngineClient):
60
self.client_aborted_requests: List[str] = []
61
62
# Processor (converts Inputs --> EngineCoreRequests).
63
- self.processor = Processor(vllm_config.model_config,
64
- vllm_config.lora_config, self.tokenizer,
65
- input_registry)
63
+ self.processor = Processor(
64
+ model_config=vllm_config.model_config,
65
+ cache_config=vllm_config.cache_config,
66
+ lora_config=vllm_config.lora_config,
67
+ tokenizer=self.tokenizer,
68
+ input_registry=input_registry,
69
+ )
70
71
# Detokenizer (converts EngineCoreOutputs --> RequestOutput).
72
self.detokenizer = Detokenizer(
vllm/v1/engine/core.py
+4
-4
@@ -65,7 +65,8 @@ class EngineCore:
65
66
self._last_logging_time = time.time()
67
68
- self.mm_input_mapper_server = MMInputMapperServer()
68
+ self.mm_input_mapper_server = MMInputMapperServer(
69
+ vllm_config.model_config)
70
71
def _initialize_kv_caches(self,
72
cache_config: CacheConfig) -> Tuple[int, int]:
@@ -98,9 +99,8 @@ class EngineCore:
99
# MM mapper, so anything that has a hash must have a HIT cache
100
# entry here as well.
101
assert request.mm_inputs is not None
101
- request.mm_inputs, request.mm_hashes = (
102
- self.mm_input_mapper_server.process_inputs(
103
- request.mm_inputs, request.mm_hashes))
102
+ request.mm_inputs = self.mm_input_mapper_server.process_inputs(
103
+ request.mm_inputs, request.mm_hashes)
104
105
req = Request.from_engine_core_request(request)
106
vllm/v1/engine/llm_engine.py
+6
-3
@@ -55,9 +55,12 @@ class LLMEngine:
55
self.tokenizer.ping()
56
57
# Processor (convert Inputs --> EngineCoreRequests)
58
- self.processor = Processor(vllm_config.model_config,
59
- vllm_config.lora_config, self.tokenizer,
60
- input_registry, mm_registry)
58
+ self.processor = Processor(model_config=vllm_config.model_config,
59
+ cache_config=vllm_config.cache_config,
60
+ lora_config=vllm_config.lora_config,
61
+ tokenizer=self.tokenizer,
62
+ input_registry=input_registry,
63
+ mm_registry=mm_registry)
64
65
# Detokenizer (converts EngineCoreOutputs --> RequestOutput)
66
self.detokenizer = Detokenizer(
vllm/v1/engine/mm_input_mapper.py
+15
-18
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional, Tuple
1
+from typing import Any, Dict, List, Optional
2
3
import PIL
4
from blake3 import blake3
@@ -42,6 +42,8 @@ class MMInputMapperClient:
42
model_config)
43
self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45
+ # Init cache
46
+ self.use_cache = model_config.mm_cache_preprocessor
47
self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
48
49
# DEBUG: Set to None to disable
@@ -61,7 +63,7 @@ class MMInputMapperClient:
63
mm_hashes: Optional[List[str]],
64
mm_processor_kwargs: Optional[Dict[str, Any]],
65
precomputed_mm_inputs: Optional[List[MultiModalKwargs]],
64
- ) -> Tuple[List[MultiModalKwargs], Optional[List[str]]]:
66
+ ) -> List[MultiModalKwargs]:
67
if precomputed_mm_inputs is None:
68
image_inputs = mm_data["image"]
69
if not isinstance(image_inputs, list):
@@ -70,26 +72,21 @@ class MMInputMapperClient:
72
else:
73
num_inputs = len(precomputed_mm_inputs)
74
73
- # Check if hash is enabled
74
- use_hash = mm_hashes is not None
75
- if use_hash:
75
+ # Sanity
76
+ if self.use_cache:
77
assert mm_hashes is not None
77
- assert num_inputs == len(
78
- mm_hashes), "num_inputs = {} len(mm_hashes) = {}".format(
79
- num_inputs, len(mm_hashes))
78
+ assert num_inputs == len(mm_hashes)
79
80
# Process each image input separately, so that later we can schedule
81
# them in a fine-grained manner.
82
# Apply caching (if enabled) and reuse precomputed inputs (if provided)
84
- ret_hashes: Optional[List[str]] = [] if use_hash else None
83
ret_inputs: List[MultiModalKwargs] = []
84
for input_id in range(num_inputs):
85
if self.mm_debug_cache_hit_ratio_steps is not None:
86
self.cache_hit_ratio(self.mm_debug_cache_hit_ratio_steps)
87
90
- mm_hash = None
88
mm_input = None
92
- if use_hash:
89
+ if self.use_cache:
90
assert mm_hashes is not None
91
mm_hash = mm_hashes[input_id]
92
mm_input = self.mm_cache.get(mm_hash)
@@ -106,7 +103,7 @@ class MMInputMapperClient:
103
mm_processor_kwargs=mm_processor_kwargs,
104
)
105
109
- if use_hash:
106
+ if self.use_cache:
107
# Add to cache
108
assert mm_hash is not None
109
self.mm_cache.put(mm_hash, mm_input)
@@ -114,18 +111,15 @@ class MMInputMapperClient:
111
self.mm_cache_hits += 1
112
mm_input = None # Avoids sending mm_input to Server
113
117
- if use_hash:
118
- assert mm_hash is not None
119
- assert ret_hashes is not None
120
- ret_hashes.append(mm_hash)
114
ret_inputs.append(mm_input)
115
123
- return ret_inputs, ret_hashes
116
+ return ret_inputs
117
118
119
class MMInputMapperServer:
120
128
- def __init__(self, ):
121
+ def __init__(self, model_config):
122
+ self.use_cache = model_config.mm_cache_preprocessor
123
self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
124
125
def process_inputs(
@@ -135,6 +129,9 @@ class MMInputMapperServer:
129
) -> List[MultiModalKwargs]:
130
assert len(mm_inputs) == len(mm_hashes)
131
132
+ if not self.use_cache:
133
+ return mm_inputs
134
+
135
full_mm_inputs = []
136
for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
137
assert mm_hash is not None
vllm/v1/engine/processor.py
+7
-5
@@ -1,7 +1,7 @@
1
import time
2
from typing import Any, Dict, Mapping, Optional, Tuple, Union
3
4
-from vllm.config import LoRAConfig, ModelConfig
4
+from vllm.config import CacheConfig, LoRAConfig, ModelConfig
5
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
6
PromptType, SingletonInputsAdapter)
7
from vllm.inputs.parse import is_encoder_decoder_inputs
@@ -23,6 +23,7 @@ class Processor:
23
def __init__(
24
self,
25
model_config: ModelConfig,
26
+ cache_config: CacheConfig,
27
lora_config: Optional[LoRAConfig],
28
tokenizer: BaseTokenizerGroup,
29
input_registry: InputRegistry = INPUT_REGISTRY,
@@ -45,8 +46,9 @@ class Processor:
46
self.mm_input_mapper_client = MMInputMapperClient(model_config)
47
48
# Multi-modal hasher (for images)
48
- self.mm_hasher = MMHasher(
49
- ) if model_config.mm_cache_preprocessor else None
49
+ self.use_hash = model_config.mm_cache_preprocessor or \
50
+ cache_config.enable_prefix_caching
51
+ self.mm_hasher = MMHasher()
52
53
# TODO: run in an ThreadpoolExecutor or BackgroundProcess.
54
# This ideally should releases the GIL, so we should not block the
@@ -77,7 +79,7 @@ class Processor:
79
80
# Compute MM hashes (if enabled)
81
mm_hashes = None
80
- if self.mm_hasher is not None:
82
+ if self.use_hash:
83
mm_hashes = self.mm_hasher.hash(prompt)
84
85
# Process inputs.
@@ -118,7 +120,7 @@ class Processor:
120
# Apply MM mapper
121
mm_inputs = None
122
if len(decoder_inputs.multi_modal_data) > 0:
121
- mm_inputs, mm_hashes = self.mm_input_mapper_client.process_inputs(
123
+ mm_inputs = self.mm_input_mapper_client.process_inputs(
124
decoder_inputs.multi_modal_data, mm_hashes,
125
decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
126
vllm/v1/request.py
+23
-1
@@ -1,5 +1,5 @@
1
import enum
2
-from typing import List, Optional, Union
2
+from typing import TYPE_CHECKING, List, Optional, Union
3
4
from vllm.inputs import DecoderOnlyInputs, SingletonInputsAdapter, token_inputs
5
from vllm.lora.request import LoRARequest
@@ -9,6 +9,9 @@ from vllm.sequence import RequestMetrics
9
from vllm.v1.engine import EngineCoreRequest
10
from vllm.v1.utils import ConstantList
11
12
+if TYPE_CHECKING:
13
+ from vllm.v1.core.kv_cache_utils import BlockHashType
14
+
15
16
class Request:
17
@@ -45,6 +48,7 @@ class Request:
48
self._all_token_ids: List[int] = self.prompt_token_ids.copy()
49
self.num_computed_tokens = 0
50
51
+ # Multi-modal input metadata.
52
mm_positions = self.inputs.multi_modal_placeholders
53
if mm_positions:
54
# FIXME(woosuk): Support other modalities.
@@ -56,6 +60,12 @@ class Request:
60
if self.inputs.multi_modal_inputs:
61
self.mm_inputs = self.inputs.multi_modal_inputs
62
63
+ self.mm_hashes: List[str] = self.inputs.multi_modal_hashes
64
+
65
+ # Cache the computed kv block hashes of the request to avoid
66
+ # recomputing.
67
+ self._kv_block_hashes: List[BlockHashType] = []
68
+
69
@classmethod
70
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
71
return cls(
@@ -65,6 +75,7 @@ class Request:
75
prompt=request.prompt,
76
multi_modal_data=None,
77
multi_modal_inputs=request.mm_inputs,
78
+ multi_modal_hashes=request.mm_hashes,
79
multi_modal_placeholders=request.mm_placeholders,
80
mm_processor_kwargs=None,
81
),
@@ -121,6 +132,17 @@ class Request:
132
num_tokens = self.mm_positions[input_id]["length"]
133
return num_tokens
134
135
+ @property
136
+ def kv_block_hashes(self) -> ConstantList["BlockHashType"]:
137
+ # Prevent directly appending to the kv_block_hashes.
138
+ return ConstantList(self._kv_block_hashes)
139
+
140
+ def set_kv_block_hashes(self, value: List["BlockHashType"]) -> None:
141
+ self._kv_block_hashes = value
142
+
143
+ def append_kv_block_hashes(self, block_hash: "BlockHashType") -> None:
144
+ self._kv_block_hashes.append(block_hash)
145
+
146
147
class RequestStatus(enum.IntEnum):
148
"""Status of a request."""