@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf8717ebae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Prefix caching for vision language models (#11187)

Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>

Cody Yu committed Dec 17, 2024 at 16:37 UTC bf8717ebaea8d74279df84fbe127ad22cf62e219
14 files changed +342 -98
tests/v1/core/test_prefix_caching.py
+86 -2
@@ -2,16 +2,23 @@
2 import pytest
3
4 from vllm.inputs import token_inputs
5 +from vllm.multimodal.inputs import PlaceholderRange
6 from vllm.sampling_params import SamplingParams
7 from vllm.utils import cdiv
8 from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
9 from vllm.v1.core.kv_cache_utils import KVCacheBlock, hash_block_tokens
10
11
11 -def make_request(request_id, prompt_token_ids):
12 +def make_request(request_id,
13 + prompt_token_ids,
14 + mm_positions=None,
15 + mm_hashes=None):
16 return Request(
17 request_id=request_id,
14 - inputs=token_inputs(prompt_token_ids=prompt_token_ids),
18 + inputs=token_inputs(prompt_token_ids=prompt_token_ids,
19 + multi_modal_placeholders={"image": mm_positions}
20 + if mm_positions else None,
21 + multi_modal_hashes=mm_hashes),
22 sampling_params=SamplingParams(max_tokens=17),
23 eos_token_id=100,
24 arrival_time=0,
@@ -38,6 +45,7 @@ def test_prefill():
45 all_token_ids = common_token_ids + unique_token_ids
46 req0 = make_request("0", all_token_ids)
47 computed_blocks = manager.get_computed_blocks(req0)
48 + assert len(req0.kv_block_hashes) == 3
49 assert not computed_blocks
50 blocks = manager.allocate_slots(req0, 55, computed_blocks)
51 assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
@@ -61,6 +69,7 @@ def test_prefill():
69 unique_token_ids = [3] * 5
70 req1 = make_request("1", common_token_ids + unique_token_ids)
71 computed_blocks = manager.get_computed_blocks(req1)
72 + assert len(req1.kv_block_hashes) == 3
73 assert [b.block_id for b in computed_blocks] == [0, 1, 2]
74 num_new_tokens = 53 - 3 * 16
75 blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
@@ -90,6 +99,7 @@ def test_prefill():
99 unique_token_ids = [3] * 6
100 req2 = make_request("2", common_token_ids + unique_token_ids)
101 computed_block = manager.get_computed_blocks(req2)
102 + assert len(req2.kv_block_hashes) == 3
103 assert [b.block_id for b in computed_block] == [0, 1, 2]
104 num_new_tokens = 53 - 3 * 16
105 blocks = manager.allocate_slots(req2, num_new_tokens, computed_blocks)
@@ -416,3 +426,77 @@ def test_cache_blocks():
426 )
427 assert len(manager.cached_block_hash_to_block) == 3
428 assert blocks[0].block_hash is not None
429 +
430 +
431 +def test_mm_prefix_caching():
432 + """
433 + This tests that the multi-modal prefix caching is correct.
434 + """
435 + manager = KVCacheManager(
436 + block_size=16,
437 + num_gpu_blocks=10,
438 + max_model_len=8192,
439 + sliding_window=None,
440 + enable_caching=True,
441 + num_preallocate_tokens=16,
442 + )
443 +
444 + # Common prompt tokens (T is text tokens and P is image placeholder tokens)
445 + # [T,...,T, P0,...,P0], [P0,...,P0,T,...,T,P1,...,P1], [P1,...,P1]
446 + common_token_ids = list(range(10)) + [-1] * 6
447 + common_token_ids += [-1] * 4 + list(range(10, 20)) + [-1] * 2
448 + common_token_ids += [-1] * 16
449 +
450 + common_mm_positions = [
451 + PlaceholderRange(offset=11, length=10),
452 + PlaceholderRange(offset=30, length=18),
453 + ]
454 + common_mm_hashes = ["aaa", "bbb"]
455 +
456 + # A unique image plus some text tokens.
457 + unique_token_ids = [-1] * 7 + [100] * 4
458 + all_token_ids = common_token_ids + unique_token_ids
459 + mm_positions = common_mm_positions + [
460 + PlaceholderRange(offset=48, length=7)
461 + ]
462 + mm_hashes = common_mm_hashes + ["ccc"]
463 + req0 = make_request("0",
464 + all_token_ids,
465 + mm_positions=mm_positions,
466 + mm_hashes=mm_hashes)
467 + computed_blocks = manager.get_computed_blocks(req0)
468 +
469 + # Completed block should have hashes with extra keys.
470 + assert not computed_blocks
471 + assert len(req0.kv_block_hashes) == 3
472 + assert req0.kv_block_hashes[0].extra_keys == (("aaa", 0), )
473 + assert req0.kv_block_hashes[1].extra_keys == (("aaa", 5), ("bbb", 0))
474 + assert req0.kv_block_hashes[2].extra_keys == (("bbb", 2), )
475 +
476 + blocks = manager.allocate_slots(req0, 59, computed_blocks)
477 + assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
478 + req0.num_computed_tokens = 59
479 +
480 + # Append slots without allocating a new block.
481 + for _ in range(5):
482 + req0.append_output_token_ids(8)
483 + new_blocks = manager.append_slots(req0, 5)
484 + assert new_blocks is not None and len(new_blocks) == 0
485 +
486 + # The just completed block should have hashes with extra keys.
487 + assert len(req0.kv_block_hashes) == 4
488 + assert req0.kv_block_hashes[3].extra_keys == (("ccc", 0), )
489 +
490 + # Cache hit.
491 + unique_token_ids = [-1] * 7 + [200] * 5
492 + all_token_ids = common_token_ids + unique_token_ids
493 + mm_positions = common_mm_positions + [
494 + PlaceholderRange(offset=48, length=7)
495 + ]
496 + mm_hashes = common_mm_hashes + ["ccc"]
497 + req1 = make_request("1",
498 + all_token_ids,
499 + mm_positions=mm_positions,
500 + mm_hashes=mm_hashes)
501 + computed_blocks = manager.get_computed_blocks(req1)
502 + assert len(computed_blocks) == 3
tests/v1/engine/test_engine_args.py
-15
@@ -31,14 +31,6 @@ def test_prefix_caching_from_cli():
31 assert engine_args.enable_prefix_caching
32
33
34 -def test_defaults():
35 - engine_args = EngineArgs(model="facebook/opt-125m")
36 -
37 - # Assert V1 defaults
38 - assert (engine_args.enable_prefix_caching
39 - ), "V1 turns on prefix caching by default"
40 -
41 -
34 def test_defaults_with_usage_context():
35 engine_args = EngineArgs(model="facebook/opt-125m")
36 vllm_config: VllmConfig = engine_args.create_engine_config(
@@ -52,10 +44,3 @@ def test_defaults_with_usage_context():
44 UsageContext.OPENAI_API_SERVER)
45 assert vllm_config.scheduler_config.max_num_seqs == 1024
46 assert vllm_config.scheduler_config.max_num_batched_tokens == 2048
55 -
56 -
57 -def test_prefix_cache_disabled_with_multimodel():
58 - engine_args = EngineArgs(model="llava-hf/llava-1.5-7b-hf")
59 -
60 - vllm_config = engine_args.create_engine_config(UsageContext.LLM_CLASS)
61 - assert not vllm_config.cache_config.enable_prefix_caching
vllm/engine/arg_utils.py
+14 -13
@@ -205,6 +205,7 @@ class EngineArgs:
205 # by user.
206 if self.enable_prefix_caching is None:
207 self.enable_prefix_caching = bool(envs.VLLM_USE_V1)
208 +
209 # Override max_num_seqs if it's not set by user.
210 if self.max_num_seqs is None:
211 self.max_num_seqs = 256 if not envs.VLLM_USE_V1 else 1024
@@ -1026,11 +1027,11 @@ class EngineArgs:
1027 device_config = DeviceConfig(device=self.device)
1028 model_config = self.create_model_config()
1029
1029 - if model_config.is_multimodal_model:
1030 - if self.enable_prefix_caching:
1031 - logger.warning(
1032 - "--enable-prefix-caching is currently not "
1033 - "supported for multimodal models and has been disabled.")
1030 + if (model_config.is_multimodal_model and not envs.VLLM_USE_V1
1031 + and self.enable_prefix_caching):
1032 + logger.warning("--enable-prefix-caching is currently not "
1033 + "supported for multimodal models in v0 and "
1034 + "has been disabled.")
1035 self.enable_prefix_caching = False
1036
1037 cache_config = CacheConfig(
@@ -1249,11 +1250,14 @@ class EngineArgs:
1250 # When no user override, set the default values based on the usage
1251 # context.
1252 # TODO(woosuk): Tune the default values for different hardware.
1252 - if self.max_num_batched_tokens is None:
1253 - if usage_context == UsageContext.LLM_CLASS:
1254 - self.max_num_batched_tokens = 8192
1255 - elif usage_context == UsageContext.OPENAI_API_SERVER:
1256 - self.max_num_batched_tokens = 2048
1253 + default_max_num_batched_tokens = {
1254 + UsageContext.LLM_CLASS: 8192,
1255 + UsageContext.OPENAI_API_SERVER: 2048,
1256 + }
1257 + if (self.max_num_batched_tokens is None
1258 + and usage_context in default_max_num_batched_tokens):
1259 + self.max_num_batched_tokens = default_max_num_batched_tokens[
1260 + usage_context]
1261 logger.warning(
1262 "Setting max_num_batched_tokens to %d for %s usage context.",
1263 self.max_num_batched_tokens, usage_context.value)
@@ -1263,9 +1267,6 @@ class EngineArgs:
1267 Override the EngineConfig's configs based on the usage context for V1.
1268 """
1269 assert envs.VLLM_USE_V1, "V1 is not enabled"
1266 - if engine_config.model_config.is_multimodal_model:
1267 - # TODO (ywang96): Enable APC by default when VLM supports it.
1268 - assert not engine_config.cache_config.enable_prefix_caching
1270
1271
1272 @dataclass
vllm/inputs/data.py
+20
@@ -162,6 +162,11 @@ class TokenInputs(TypedDict):
162 Placeholder ranges for the multi-modal data.
163 """
164
165 + multi_modal_hashes: NotRequired[List[str]]
166 + """
167 + The hashes of the multi-modal data.
168 + """
169 +
170 mm_processor_kwargs: NotRequired[Dict[str, Any]]
171 """
172 Optional multi-modal processor kwargs to be forwarded to the
@@ -177,6 +182,7 @@ def token_inputs(
182 prompt: Optional[str] = None,
183 multi_modal_data: Optional["MultiModalDataDict"] = None,
184 multi_modal_inputs: Optional["MultiModalKwargs"] = None,
185 + multi_modal_hashes: Optional[List[str]] = None,
186 multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
187 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
188 ) -> TokenInputs:
@@ -191,6 +197,8 @@ def token_inputs(
197 inputs["multi_modal_data"] = multi_modal_data
198 if multi_modal_inputs is not None:
199 inputs["multi_modal_inputs"] = multi_modal_inputs
200 + if multi_modal_hashes is not None:
201 + inputs["multi_modal_hashes"] = multi_modal_hashes
202 if multi_modal_placeholders is not None:
203 inputs["multi_modal_placeholders"] = multi_modal_placeholders
204 if mm_processor_kwargs is not None:
@@ -295,6 +303,18 @@ class SingletonInputsAdapter:
303
304 assert_never(inputs)
305
306 + @cached_property
307 + def multi_modal_hashes(self) -> List[str]:
308 + inputs = self.inputs
309 +
310 + if inputs["type"] == "token":
311 + return inputs.get("multi_modal_hashes", [])
312 +
313 + if inputs["type"] == "multimodal":
314 + return inputs.get("mm_hashes", [])
315 +
316 + assert_never(inputs)
317 +
318 @cached_property
319 def multi_modal_placeholders(self) -> "MultiModalPlaceholderDict":
320 inputs = self.inputs
vllm/multimodal/inputs.py
+3
@@ -215,6 +215,9 @@ class MultiModalInputsV2(TypedDict):
215 mm_kwargs: MultiModalKwargs
216 """Keyword arguments to be directly passed to the model after batching."""
217
218 + mm_hashes: NotRequired[List[str]]
219 + """The hashes of the multi-modal data."""
220 +
221 mm_placeholders: MultiModalPlaceholderDict
222 """
223 For each modality, information about the placeholder tokens in
vllm/v1/core/kv_cache_manager.py
+50 -24
@@ -4,7 +4,9 @@ from typing import Dict, Iterable, List, Optional
4 from vllm.logger import init_logger
5 from vllm.utils import cdiv
6 from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
7 - KVCacheBlock, hash_block_tokens,
7 + KVCacheBlock,
8 + generate_block_hash_extra_keys,
9 + hash_block_tokens,
10 hash_request_tokens)
11 from vllm.v1.request import Request
12
@@ -83,10 +85,12 @@ class KVCacheManager:
85
86 computed_blocks = []
87
86 - # TODO(rickyx): potentially we could cache this so we don't have to
87 - # recompute it every time.
88 - block_hashes = hash_request_tokens(self.block_size,
89 - request.all_token_ids)
88 + # The block hashes for the request may already be computed
89 + # if the request was preempted and resumed.
90 + if not request.kv_block_hashes:
91 + request.set_kv_block_hashes(
92 + hash_request_tokens(self.block_size, request))
93 + block_hashes = request.kv_block_hashes
94
95 for block_hash in block_hashes:
96 # block_hashes is a chain of block hashes. If a block hash is not
@@ -242,14 +246,16 @@ class KVCacheManager:
246 num_computed_tokens = len(computed_blocks) * self.block_size
247 num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
248
245 - self._cache_full_blocks(
246 - request=request,
247 - blk_start_idx=len(computed_blocks),
248 - # The new full blocks are the full blocks that are not computed.
249 - full_blocks=self.req_to_blocks[request.request_id]
250 - [len(computed_blocks):num_full_blocks],
251 - prev_block=computed_blocks[-1] if computed_blocks else None,
252 - )
249 + new_full_blocks = self.req_to_blocks[
250 + request.request_id][len(computed_blocks):num_full_blocks]
251 + if new_full_blocks:
252 + self._cache_full_blocks(
253 + request=request,
254 + blk_start_idx=len(computed_blocks),
255 + # The new full blocks are the full blocks that are not computed.
256 + full_blocks=new_full_blocks,
257 + prev_block=computed_blocks[-1] if computed_blocks else None,
258 + )
259
260 return new_blocks
261
@@ -376,6 +382,8 @@ class KVCacheManager:
382 full_blocks: The list of blocks to update hash metadata.
383 prev_block: The previous block in the chain.
384 """
385 + num_cached_block_hashes = len(request.kv_block_hashes)
386 +
387 # Update the new blocks with the block hashes through the chain.
388 prev_block_hash_value = None
389 if prev_block is not None:
@@ -387,17 +395,35 @@ class KVCacheManager:
395 for i, blk in enumerate(full_blocks):
396 blk_idx = blk_start_idx + i
397
390 - block_tokens = request.all_token_ids[blk_idx *
391 - self.block_size:(blk_idx +
392 - 1) *
393 - self.block_size]
394 - assert len(block_tokens) == self.block_size, (
395 - f"Expected {self.block_size} tokens, got {len(block_tokens)} "
396 - f"at {blk_idx}th block for request "
397 - f"{request.request_id}({request})")
398 -
399 - # Compute the hash of the current block.
400 - block_hash = hash_block_tokens(prev_block_hash_value, block_tokens)
398 + if blk_idx < num_cached_block_hashes:
399 + # The block hash may already be computed in
400 + # "get_computed_blocks" if the tokens are not generated by
401 + # this request (either the prompt tokens or the previously
402 + # generated tokens with preemption). In this case we simply
403 + # reuse the block hash.
404 + block_hash = request.kv_block_hashes[blk_idx]
405 + else:
406 + # Otherwise compute the block hash and cache it in the request
407 + # in case it will be preempted in the future.
408 + start_token_idx = blk_idx * self.block_size
409 + end_token_idx = (blk_idx + 1) * self.block_size
410 + block_tokens = request.all_token_ids[
411 + start_token_idx:end_token_idx]
412 + assert len(block_tokens) == self.block_size, (
413 + f"Expected {self.block_size} tokens, got "
414 + f"{len(block_tokens)} at {blk_idx}th block for request "
415 + f"{request.request_id}({request})")
416 +
417 + # Generate extra keys for multi-modal inputs. Note that since
418 + # we reach to this branch only when the block is completed with
419 + # generated tokens, we only need to consider the last mm input.
420 + extra_keys, _ = generate_block_hash_extra_keys(
421 + request, start_token_idx, end_token_idx, -1)
422 +
423 + # Compute the hash of the current block.
424 + block_hash = hash_block_tokens(prev_block_hash_value,
425 + block_tokens, extra_keys)
426 + request.append_kv_block_hashes(block_hash)
427
428 # Update and added the full block to the cache.
429 blk.block_hash = block_hash
vllm/v1/core/kv_cache_utils.py
+105 -10
@@ -1,20 +1,25 @@
1 """KV-Cache Utilities."""
2 from collections.abc import Sequence
3 from dataclasses import dataclass
4 -from typing import List, NamedTuple, Optional, Tuple
4 +from typing import Any, List, NamedTuple, Optional, Tuple
5
6 from vllm.logger import init_logger
7 +from vllm.v1.request import Request
8
9 logger = init_logger(__name__)
10
11
12 class BlockHashType(NamedTuple):
12 - """Hash value of a block and the token IDs in the block.
13 - The reason we keep a tuple of token IDs is to make sure no hash
14 - collision happens when the hash value is the same.
13 + """Hash value of a block (int), the token IDs in the block, and extra keys.
14 + The reason we keep a tuple of token IDs and extra keys is to make sure
15 + no hash collision happens when the hash value is the same.
16 """
17 + # Hash value of the block in an integer.
18 hash_value: int
19 + # Token IDs in the block.
20 token_ids: Tuple[int, ...]
21 + # Extra keys for the block.
22 + extra_keys: Optional[Any] = None
23
24
25 @dataclass
@@ -159,8 +164,80 @@ class FreeKVCacheBlockQueue:
164 return ret
165
166
162 -def hash_block_tokens(parent_block_hash: Optional[int],
163 - curr_block_token_ids: Sequence[int]) -> BlockHashType:
167 +def generate_block_hash_extra_keys(
168 + request: Request, start_token_idx: int, end_token_idx: int,
169 + start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:
170 + """Generate extra keys for the block hash. The extra keys can come from
171 + the multi-modal inputs and request specific metadata (e.g., LoRA ID).
172 + For multi-modal inputs, the extra keys are (mm_hash, start_offset) that
173 + indicate a mm input contained in the block and its starting offset in
174 + the block tokens.
175 +
176 + Args:
177 + request: The request object.
178 + start_token_idx: The start token index of the block.
179 + end_token_idx: The end token index of the block.
180 + start_mm_idx: The start multi-modal index of the block.
181 +
182 + Returns:
183 + A tuple of extra keys and the next multi-modal index.
184 + """
185 +
186 + mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
187 + if not mm_positions:
188 + return None, start_mm_idx
189 +
190 + if mm_positions and len(mm_positions) != len(mm_hashes):
191 + raise ValueError(
192 + "The number of multi-modal positions and hashes must match. This "
193 + "is likely because you do not enable MM preprocessor hashing. "
194 + "Please set mm_cache_preprocessor=True.")
195 +
196 + # Note that we assume mm_positions is sorted by offset.
197 + # We do not need to check all mm inputs if the start token index is out of
198 + # range. This usually happens in the late prefill phase and decoding phase.
199 + if mm_positions[-1]["offset"] + mm_positions[-1][
200 + "length"] < start_token_idx:
201 + return None, start_mm_idx
202 +
203 + # Support start_mm_idx == -1 to indicate the last mm input.
204 + if start_mm_idx < 0:
205 + assert -start_mm_idx <= len(mm_positions)
206 + start_mm_idx = len(mm_positions) + start_mm_idx
207 +
208 + extra_keys = []
209 + curr_mm_idx = start_mm_idx
210 + while mm_positions and curr_mm_idx < len(mm_positions):
211 + assert mm_hashes[curr_mm_idx] is not None
212 + offset = mm_positions[curr_mm_idx]["offset"]
213 + length = mm_positions[curr_mm_idx]["length"]
214 + if end_token_idx > offset:
215 + if start_token_idx > offset + length:
216 + # This block has passed the current mm input.
217 + curr_mm_idx += 1
218 + continue
219 +
220 + # The block contains the current mm input.
221 + mm_start = max(0, start_token_idx - offset)
222 + extra_keys.append((mm_hashes[curr_mm_idx], mm_start))
223 + if end_token_idx >= offset + length:
224 + # If this block contains the end of the current mm input,
225 + # move to the next mm input as this block may also contain
226 + # the next mm input.
227 + curr_mm_idx += 1
228 + else:
229 + # Otherwise this block is done with mm inputs.
230 + break
231 + else:
232 + # This block has not reached the current mm input.
233 + break
234 + return tuple(extra_keys), curr_mm_idx
235 +
236 +
237 +def hash_block_tokens(
238 + parent_block_hash: Optional[int],
239 + curr_block_token_ids: Sequence[int],
240 + extra_keys: Optional[Tuple[Any, ...]] = None) -> BlockHashType:
241 """Computes a hash value corresponding to the contents of a block and
242 the contents of the preceding block(s). The hash value is used for
243 prefix caching. We use LRU cache for this function to avoid recomputing
@@ -174,27 +251,39 @@ def hash_block_tokens(parent_block_hash: Optional[int],
251 if this is the first block.
252 curr_block_token_ids: A list of token ids in the current
253 block. The current block is assumed to be full.
254 + extra_keys: Extra keys for the block.
255
256 Returns:
257 The hash value of the block and the token ids in the block.
258 The entire tuple is used as the hash key of the block.
259 """
260 return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
183 - tuple(curr_block_token_ids))
261 + tuple(curr_block_token_ids), extra_keys)
262
263
264 def hash_request_tokens(block_size: int,
187 - token_ids: Sequence[int]) -> List[BlockHashType]:
265 + request: Request) -> List[BlockHashType]:
266 """Computes hash values of a chain of blocks given a sequence of
267 token IDs. The hash value is used for prefix caching.
268
269 Args:
270 block_size: The size of each block.
193 - token_ids: A sequence of token ids in the request.
271 + request: The request object.
272
273 Returns:
274 The list of computed hash values.
275 """
276 + token_ids = request.all_token_ids
277 + mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
278 + if mm_positions and len(mm_positions) != len(mm_hashes):
279 + raise ValueError(
280 + "The number of multi-modal positions and hashes must match.")
281 +
282 + # TODO: Extend this to support other features such as LoRA.
283 + need_extra_keys = bool(mm_positions)
284 + extra_keys = None
285 + curr_mm_idx = 0
286 +
287 ret = []
288 parent_block_hash_value = None
289 for start in range(0, len(token_ids), block_size):
@@ -203,8 +292,14 @@ def hash_request_tokens(block_size: int,
292 # Do not hash the block if it is not full.
293 if len(block_token_ids) < block_size:
294 break
295 +
296 + # Add extra keys if the block is a multi-modal block.
297 + if need_extra_keys:
298 + extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
299 + request, start, end, curr_mm_idx)
300 +
301 block_hash = hash_block_tokens(parent_block_hash_value,
207 - block_token_ids)
302 + block_token_ids, extra_keys)
303 ret.append(block_hash)
304 parent_block_hash_value = block_hash.hash_value
305 return ret
vllm/v1/core/scheduler.py
+2
@@ -516,6 +516,7 @@ class NewRequestData:
516 prompt_token_ids: List[int]
517 prompt: Optional[str]
518 mm_inputs: List["MultiModalKwargs"]
519 + mm_hashes: List[str]
520 mm_positions: List["PlaceholderRange"]
521 sampling_params: SamplingParams
522 block_ids: List[int]
@@ -533,6 +534,7 @@ class NewRequestData:
534 prompt_token_ids=request.prompt_token_ids,
535 prompt=request.prompt,
536 mm_inputs=request.mm_inputs,
537 + mm_hashes=request.mm_hashes,
538 mm_positions=request.mm_positions,
539 sampling_params=request.sampling_params,
540 block_ids=block_ids,
vllm/v1/engine/async_llm.py
+7 -3
@@ -60,9 +60,13 @@ class AsyncLLM(EngineClient):
60 self.client_aborted_requests: List[str] = []
61
62 # Processor (converts Inputs --> EngineCoreRequests).
63 - self.processor = Processor(vllm_config.model_config,
64 - vllm_config.lora_config, self.tokenizer,
65 - input_registry)
63 + self.processor = Processor(
64 + model_config=vllm_config.model_config,
65 + cache_config=vllm_config.cache_config,
66 + lora_config=vllm_config.lora_config,
67 + tokenizer=self.tokenizer,
68 + input_registry=input_registry,
69 + )
70
71 # Detokenizer (converts EngineCoreOutputs --> RequestOutput).
72 self.detokenizer = Detokenizer(
vllm/v1/engine/core.py
+4 -4
@@ -65,7 +65,8 @@ class EngineCore:
65
66 self._last_logging_time = time.time()
67
68 - self.mm_input_mapper_server = MMInputMapperServer()
68 + self.mm_input_mapper_server = MMInputMapperServer(
69 + vllm_config.model_config)
70
71 def _initialize_kv_caches(self,
72 cache_config: CacheConfig) -> Tuple[int, int]:
@@ -98,9 +99,8 @@ class EngineCore:
99 # MM mapper, so anything that has a hash must have a HIT cache
100 # entry here as well.
101 assert request.mm_inputs is not None
101 - request.mm_inputs, request.mm_hashes = (
102 - self.mm_input_mapper_server.process_inputs(
103 - request.mm_inputs, request.mm_hashes))
102 + request.mm_inputs = self.mm_input_mapper_server.process_inputs(
103 + request.mm_inputs, request.mm_hashes)
104
105 req = Request.from_engine_core_request(request)
106
vllm/v1/engine/llm_engine.py
+6 -3
@@ -55,9 +55,12 @@ class LLMEngine:
55 self.tokenizer.ping()
56
57 # Processor (convert Inputs --> EngineCoreRequests)
58 - self.processor = Processor(vllm_config.model_config,
59 - vllm_config.lora_config, self.tokenizer,
60 - input_registry, mm_registry)
58 + self.processor = Processor(model_config=vllm_config.model_config,
59 + cache_config=vllm_config.cache_config,
60 + lora_config=vllm_config.lora_config,
61 + tokenizer=self.tokenizer,
62 + input_registry=input_registry,
63 + mm_registry=mm_registry)
64
65 # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
66 self.detokenizer = Detokenizer(
vllm/v1/engine/mm_input_mapper.py
+15 -18
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional, Tuple
1 +from typing import Any, Dict, List, Optional
2
3 import PIL
4 from blake3 import blake3
@@ -42,6 +42,8 @@ class MMInputMapperClient:
42 model_config)
43 self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45 + # Init cache
46 + self.use_cache = model_config.mm_cache_preprocessor
47 self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
48
49 # DEBUG: Set to None to disable
@@ -61,7 +63,7 @@ class MMInputMapperClient:
63 mm_hashes: Optional[List[str]],
64 mm_processor_kwargs: Optional[Dict[str, Any]],
65 precomputed_mm_inputs: Optional[List[MultiModalKwargs]],
64 - ) -> Tuple[List[MultiModalKwargs], Optional[List[str]]]:
66 + ) -> List[MultiModalKwargs]:
67 if precomputed_mm_inputs is None:
68 image_inputs = mm_data["image"]
69 if not isinstance(image_inputs, list):
@@ -70,26 +72,21 @@ class MMInputMapperClient:
72 else:
73 num_inputs = len(precomputed_mm_inputs)
74
73 - # Check if hash is enabled
74 - use_hash = mm_hashes is not None
75 - if use_hash:
75 + # Sanity
76 + if self.use_cache:
77 assert mm_hashes is not None
77 - assert num_inputs == len(
78 - mm_hashes), "num_inputs = {} len(mm_hashes) = {}".format(
79 - num_inputs, len(mm_hashes))
78 + assert num_inputs == len(mm_hashes)
79
80 # Process each image input separately, so that later we can schedule
81 # them in a fine-grained manner.
82 # Apply caching (if enabled) and reuse precomputed inputs (if provided)
84 - ret_hashes: Optional[List[str]] = [] if use_hash else None
83 ret_inputs: List[MultiModalKwargs] = []
84 for input_id in range(num_inputs):
85 if self.mm_debug_cache_hit_ratio_steps is not None:
86 self.cache_hit_ratio(self.mm_debug_cache_hit_ratio_steps)
87
90 - mm_hash = None
88 mm_input = None
92 - if use_hash:
89 + if self.use_cache:
90 assert mm_hashes is not None
91 mm_hash = mm_hashes[input_id]
92 mm_input = self.mm_cache.get(mm_hash)
@@ -106,7 +103,7 @@ class MMInputMapperClient:
103 mm_processor_kwargs=mm_processor_kwargs,
104 )
105
109 - if use_hash:
106 + if self.use_cache:
107 # Add to cache
108 assert mm_hash is not None
109 self.mm_cache.put(mm_hash, mm_input)
@@ -114,18 +111,15 @@ class MMInputMapperClient:
111 self.mm_cache_hits += 1
112 mm_input = None # Avoids sending mm_input to Server
113
117 - if use_hash:
118 - assert mm_hash is not None
119 - assert ret_hashes is not None
120 - ret_hashes.append(mm_hash)
114 ret_inputs.append(mm_input)
115
123 - return ret_inputs, ret_hashes
116 + return ret_inputs
117
118
119 class MMInputMapperServer:
120
128 - def __init__(self, ):
121 + def __init__(self, model_config):
122 + self.use_cache = model_config.mm_cache_preprocessor
123 self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
124
125 def process_inputs(
@@ -135,6 +129,9 @@ class MMInputMapperServer:
129 ) -> List[MultiModalKwargs]:
130 assert len(mm_inputs) == len(mm_hashes)
131
132 + if not self.use_cache:
133 + return mm_inputs
134 +
135 full_mm_inputs = []
136 for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
137 assert mm_hash is not None
vllm/v1/engine/processor.py
+7 -5
@@ -1,7 +1,7 @@
1 import time
2 from typing import Any, Dict, Mapping, Optional, Tuple, Union
3
4 -from vllm.config import LoRAConfig, ModelConfig
4 +from vllm.config import CacheConfig, LoRAConfig, ModelConfig
5 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
6 PromptType, SingletonInputsAdapter)
7 from vllm.inputs.parse import is_encoder_decoder_inputs
@@ -23,6 +23,7 @@ class Processor:
23 def __init__(
24 self,
25 model_config: ModelConfig,
26 + cache_config: CacheConfig,
27 lora_config: Optional[LoRAConfig],
28 tokenizer: BaseTokenizerGroup,
29 input_registry: InputRegistry = INPUT_REGISTRY,
@@ -45,8 +46,9 @@ class Processor:
46 self.mm_input_mapper_client = MMInputMapperClient(model_config)
47
48 # Multi-modal hasher (for images)
48 - self.mm_hasher = MMHasher(
49 - ) if model_config.mm_cache_preprocessor else None
49 + self.use_hash = model_config.mm_cache_preprocessor or \
50 + cache_config.enable_prefix_caching
51 + self.mm_hasher = MMHasher()
52
53 # TODO: run in an ThreadpoolExecutor or BackgroundProcess.
54 # This ideally should releases the GIL, so we should not block the
@@ -77,7 +79,7 @@ class Processor:
79
80 # Compute MM hashes (if enabled)
81 mm_hashes = None
80 - if self.mm_hasher is not None:
82 + if self.use_hash:
83 mm_hashes = self.mm_hasher.hash(prompt)
84
85 # Process inputs.
@@ -118,7 +120,7 @@ class Processor:
120 # Apply MM mapper
121 mm_inputs = None
122 if len(decoder_inputs.multi_modal_data) > 0:
121 - mm_inputs, mm_hashes = self.mm_input_mapper_client.process_inputs(
123 + mm_inputs = self.mm_input_mapper_client.process_inputs(
124 decoder_inputs.multi_modal_data, mm_hashes,
125 decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
126
vllm/v1/request.py
+23 -1
@@ -1,5 +1,5 @@
1 import enum
2 -from typing import List, Optional, Union
2 +from typing import TYPE_CHECKING, List, Optional, Union
3
4 from vllm.inputs import DecoderOnlyInputs, SingletonInputsAdapter, token_inputs
5 from vllm.lora.request import LoRARequest
@@ -9,6 +9,9 @@ from vllm.sequence import RequestMetrics
9 from vllm.v1.engine import EngineCoreRequest
10 from vllm.v1.utils import ConstantList
11
12 +if TYPE_CHECKING:
13 + from vllm.v1.core.kv_cache_utils import BlockHashType
14 +
15
16 class Request:
17
@@ -45,6 +48,7 @@ class Request:
48 self._all_token_ids: List[int] = self.prompt_token_ids.copy()
49 self.num_computed_tokens = 0
50
51 + # Multi-modal input metadata.
52 mm_positions = self.inputs.multi_modal_placeholders
53 if mm_positions:
54 # FIXME(woosuk): Support other modalities.
@@ -56,6 +60,12 @@ class Request:
60 if self.inputs.multi_modal_inputs:
61 self.mm_inputs = self.inputs.multi_modal_inputs
62
63 + self.mm_hashes: List[str] = self.inputs.multi_modal_hashes
64 +
65 + # Cache the computed kv block hashes of the request to avoid
66 + # recomputing.
67 + self._kv_block_hashes: List[BlockHashType] = []
68 +
69 @classmethod
70 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
71 return cls(
@@ -65,6 +75,7 @@ class Request:
75 prompt=request.prompt,
76 multi_modal_data=None,
77 multi_modal_inputs=request.mm_inputs,
78 + multi_modal_hashes=request.mm_hashes,
79 multi_modal_placeholders=request.mm_placeholders,
80 mm_processor_kwargs=None,
81 ),
@@ -121,6 +132,17 @@ class Request:
132 num_tokens = self.mm_positions[input_id]["length"]
133 return num_tokens
134
135 + @property
136 + def kv_block_hashes(self) -> ConstantList["BlockHashType"]:
137 + # Prevent directly appending to the kv_block_hashes.
138 + return ConstantList(self._kv_block_hashes)
139 +
140 + def set_kv_block_hashes(self, value: List["BlockHashType"]) -> None:
141 + self._kv_block_hashes = value
142 +
143 + def append_kv_block_hashes(self, block_hash: "BlockHashType") -> None:
144 + self._kv_block_hashes.append(block_hash)
145 +
146
147 class RequestStatus(enum.IntEnum):
148 """Status of a request."""