| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import multiprocessing as mp |
| 4 | from types import SimpleNamespace |
| 5 | from typing import cast |
| 6 | from unittest.mock import Mock |
| 7 | |
| 8 | import numpy as np |
| 9 | import pytest |
| 10 | import torch |
| 11 | |
| 12 | from vllm.config import ModelConfig, ParallelConfig, VllmConfig |
| 13 | from vllm.config.multimodal import MultiModalConfig |
| 14 | from vllm.multimodal.cache import ( |
| 15 | BaseMultiModalReceiverCache, |
| 16 | LruKeyReplicatedReceiverCache, |
| 17 | LruKeyReplicatedSenderCache, |
| 18 | MultiModalCacheMissError, |
| 19 | MultiModalProcessorOnlyCache, |
| 20 | ShmObjectStoreReceiverCache, |
| 21 | ShmObjectStoreSenderCache, |
| 22 | engine_receiver_cache_from_config, |
| 23 | processor_cache_from_config, |
| 24 | ) |
| 25 | from vllm.multimodal.cache.base import ( |
| 26 | MultiModalCache, |
| 27 | MultiModalProcessorCacheInItem, |
| 28 | MultiModalProcessorCacheItem, |
| 29 | MultiModalProcessorCacheItemMetadata, |
| 30 | ) |
| 31 | from vllm.multimodal.hasher import MultiModalHasher |
| 32 | from vllm.multimodal.inputs import ( |
| 33 | MultiModalBatchedField, |
| 34 | MultiModalFeatureSpec, |
| 35 | MultiModalFieldElem, |
| 36 | MultiModalKwargsItem, |
| 37 | MultiModalKwargsItems, |
| 38 | MultiModalSharedField, |
| 39 | PlaceholderRange, |
| 40 | ) |
| 41 | from vllm.multimodal.processing import PromptInsertion |
| 42 | from vllm.renderers import renderer_from_config |
| 43 | from vllm.utils.async_utils import make_async |
| 44 | from vllm.utils.mem_constants import GiB_bytes, MiB_bytes |
| 45 | from vllm.v1.engine.async_llm import AsyncLLM |
| 46 | from vllm.v1.engine.llm_engine import LLMEngine |
| 47 | from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder |
| 48 | |
| 49 | from ..models.utils import build_model_context |
| 50 | from ..utils import create_new_process_for_each_test |
| 51 | |
| 52 | pytestmark = pytest.mark.cpu_test |
| 53 | |
| 54 | |
| 55 | def _dummy_elem( |
| 56 | size: int, |
| 57 | *, |
| 58 | rng: np.random.RandomState | None = None, |
| 59 | ): |
| 60 | if rng is None: |
| 61 | data = torch.empty((size,), dtype=torch.int8) |
| 62 | else: |
| 63 | data = torch.from_numpy(rng.randint(4, size=(size,), dtype=np.int8)) |
| 64 | |
| 65 | return MultiModalFieldElem( |
| 66 | data=data, |
| 67 | field=MultiModalSharedField(batch_size=1), |
| 68 | ) |
| 69 | |
| 70 | |
| 71 | def _dummy_item( |
| 72 | size_by_key: dict[str, int], |
| 73 | *, |
| 74 | rng: np.random.RandomState | None = None, |
| 75 | ): |
| 76 | return MultiModalKwargsItem( |
| 77 | {key: _dummy_elem(size, rng=rng) for key, size in size_by_key.items()} |
| 78 | ) |
| 79 | |
| 80 | |
| 81 | def _dummy_items( |
| 82 | size_by_key_modality: dict[str, dict[str, int]], |
| 83 | *, |
| 84 | rng: np.random.RandomState | None = None, |
| 85 | ): |
| 86 | return MultiModalKwargsItems( |
| 87 | { |
| 88 | modality: [_dummy_item(size_by_key, rng=rng)] |
| 89 | for modality, size_by_key in size_by_key_modality.items() |
| 90 | } |
| 91 | ) |
| 92 | |
| 93 | |
| 94 | @pytest.mark.parametrize( |
| 95 | ("item", "expected_size"), |
| 96 | [ |
| 97 | (_dummy_item({"a1": 100}), 100), |
| 98 | (_dummy_item({"a1": 100, "a2": 110}), 210), |
| 99 | (_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501 |
| 100 | ], |
| 101 | ) |
| 102 | def test_cache_item_size(item, expected_size): |
| 103 | cache = MultiModalCache.get_lru_cache(2048, type(item)) |
| 104 | |
| 105 | cache[""] = item |
| 106 | assert cache.currsize == expected_size |
| 107 | |
| 108 | prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0) |
| 109 | |
| 110 | cache[""] = MultiModalProcessorCacheItem(item, [prompt_update]) |
| 111 | assert cache.currsize == expected_size |
| 112 | |
| 113 | cache[""] = MultiModalProcessorCacheItemMetadata(item, [prompt_update]) |
| 114 | assert cache.currsize == expected_size |
| 115 | |
| 116 | cache[""] = item.get_data() |
| 117 | assert cache.currsize == expected_size |
| 118 | |
| 119 | |
| 120 | def _create_vllm_config( |
| 121 | *, |
| 122 | mm_processor_cache_gb: float, |
| 123 | enable_ipc: bool, |
| 124 | ): |
| 125 | return VllmConfig( |
| 126 | model_config=ModelConfig( |
| 127 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 128 | mm_processor_cache_gb=mm_processor_cache_gb, |
| 129 | ), |
| 130 | parallel_config=ParallelConfig(data_parallel_size=1 if enable_ipc else 2), |
| 131 | ) |
| 132 | |
| 133 | |
| 134 | def _compare_caches( |
| 135 | config_0: VllmConfig, |
| 136 | config_1: VllmConfig, |
| 137 | *, |
| 138 | item_capacity: int = 8, |
| 139 | hit_rate: float = 0.5, |
| 140 | max_items_per_iter: int = 3, |
| 141 | is_cached_calls_per_iter: int, |
| 142 | n_iter: int = 100, |
| 143 | seed: int = 0, |
| 144 | ): |
| 145 | cache_0_p0 = processor_cache_from_config(config_0) |
| 146 | cache_0_p1 = engine_receiver_cache_from_config(config_0) |
| 147 | cache_1_p0 = processor_cache_from_config(config_1) |
| 148 | cache_1_p1 = engine_receiver_cache_from_config(config_1) |
| 149 | |
| 150 | assert config_0.model_config.multimodal_config is not None |
| 151 | assert config_1.model_config.multimodal_config is not None |
| 152 | cache_size_gb = max( |
| 153 | config_0.model_config.multimodal_config.mm_processor_cache_gb, |
| 154 | config_1.model_config.multimodal_config.mm_processor_cache_gb, |
| 155 | ) |
| 156 | item_size_gb = int(cache_size_gb / item_capacity) |
| 157 | |
| 158 | rng = np.random.RandomState(seed) |
| 159 | all_items = [ |
| 160 | _dummy_item({"key": item_size_gb}, rng=rng) |
| 161 | for _ in range(int(item_capacity / hit_rate)) |
| 162 | ] |
| 163 | all_hashes = [ |
| 164 | MultiModalHasher.hash_kwargs("blake3", item=item.get_data()) |
| 165 | for item in all_items |
| 166 | ] |
| 167 | |
| 168 | prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0) |
| 169 | |
| 170 | for it in range(n_iter): |
| 171 | num_items_to_select = rng.randint(0, max_items_per_iter) |
| 172 | item_idxs_to_select = rng.choice(len(all_items), num_items_to_select) |
| 173 | |
| 174 | selected_items = [all_items[idx] for idx in item_idxs_to_select] |
| 175 | selected_hashes = [all_hashes[idx] for idx in item_idxs_to_select] |
| 176 | |
| 177 | if cache_0_p0 is None: |
| 178 | cache_0_p0_out = selected_items |
| 179 | else: |
| 180 | for _ in range(is_cached_calls_per_iter): |
| 181 | cache_0_p0.is_cached(selected_hashes) |
| 182 | |
| 183 | cache_0_p0_out = [ |
| 184 | item |
| 185 | for item, _ in cache_0_p0.get_and_update( |
| 186 | [(item, [prompt_update]) for item in selected_items], |
| 187 | selected_hashes, |
| 188 | ) |
| 189 | ] |
| 190 | |
| 191 | if cache_1_p0 is None: |
| 192 | cache_1_p0_out = selected_items |
| 193 | else: |
| 194 | for _ in range(is_cached_calls_per_iter): |
| 195 | cache_1_p0.is_cached(selected_hashes) |
| 196 | |
| 197 | cache_1_p0_out = [ |
| 198 | item |
| 199 | for item, _ in cache_1_p0.get_and_update( |
| 200 | [(item, [prompt_update]) for item in selected_items], |
| 201 | selected_hashes, |
| 202 | ) |
| 203 | ] |
| 204 | |
| 205 | if cache_0_p1 is None: |
| 206 | cache_0_p1_out = cache_0_p0_out |
| 207 | else: |
| 208 | cache_0_p1_out = cache_0_p1.get_and_update(cache_0_p0_out, selected_hashes) |
| 209 | |
| 210 | if cache_1_p1 is None: |
| 211 | cache_1_p1_out = cache_1_p0_out |
| 212 | else: |
| 213 | cache_1_p1_out = cache_1_p1.get_and_update(cache_1_p0_out, selected_hashes) |
| 214 | |
| 215 | assert cache_0_p1_out == cache_1_p1_out, f"Failed at {it=}" |
| 216 | |
| 217 | |
| 218 | @pytest.mark.parametrize("is_cached_calls_per_iter", [1, 2, 3]) |
| 219 | def test_ipc_enable_disable_consistency(is_cached_calls_per_iter): |
| 220 | cache_size_gb = 1 / (1 << 20) |
| 221 | |
| 222 | vllm_config_ipc_enabled = _create_vllm_config( |
| 223 | mm_processor_cache_gb=cache_size_gb, |
| 224 | enable_ipc=True, |
| 225 | ) |
| 226 | vllm_config_ipc_disabled = _create_vllm_config( |
| 227 | mm_processor_cache_gb=0, |
| 228 | enable_ipc=False, |
| 229 | ) |
| 230 | vllm_config_cache_disabled = _create_vllm_config( |
| 231 | mm_processor_cache_gb=cache_size_gb, |
| 232 | enable_ipc=True, |
| 233 | ) |
| 234 | |
| 235 | _compare_caches( |
| 236 | vllm_config_ipc_enabled, |
| 237 | vllm_config_ipc_disabled, |
| 238 | is_cached_calls_per_iter=is_cached_calls_per_iter, |
| 239 | ) |
| 240 | _compare_caches( |
| 241 | vllm_config_ipc_disabled, |
| 242 | vllm_config_cache_disabled, |
| 243 | is_cached_calls_per_iter=is_cached_calls_per_iter, |
| 244 | ) |
| 245 | _compare_caches( |
| 246 | vllm_config_cache_disabled, |
| 247 | vllm_config_ipc_enabled, |
| 248 | is_cached_calls_per_iter=is_cached_calls_per_iter, |
| 249 | ) |
| 250 | |
| 251 | |
| 252 | class _StubModelConfig: |
| 253 | """Minimal model-config stand-in: the cache classes only call |
| 254 | get_multimodal_config(), so this lets us construct them directly without a |
| 255 | real model, keeping this unit test free of any Hugging Face / network lookup. |
| 256 | """ |
| 257 | |
| 258 | def __init__(self, mm_processor_cache_gb: float) -> None: |
| 259 | self._mm_config = MultiModalConfig(mm_processor_cache_gb=mm_processor_cache_gb) |
| 260 | |
| 261 | def get_multimodal_config(self) -> MultiModalConfig: |
| 262 | return self._mm_config |
| 263 | |
| 264 | |
| 265 | @pytest.mark.skip_global_cleanup |
| 266 | def test_oversized_item_is_served_uncached(): |
| 267 | """Items larger than the processor cache must not crash insert. |
| 268 | |
| 269 | cachetools.LRUCache raises ValueError("value too large") when a single |
| 270 | item exceeds maxsize. Engine profiling uses a max-size dummy item, so this |
| 271 | used to abort EngineCore startup (vllm-project/vllm#52835). Skip the |
| 272 | insert and serve the item uncached instead. |
| 273 | """ |
| 274 | # 1 KiB cache; a 4 KiB item cannot fit even if the cache is empty. |
| 275 | model_config = _StubModelConfig(mm_processor_cache_gb=1024 / GiB_bytes) |
| 276 | item = MultiModalKwargsItem.dummy(nbytes=4096) |
| 277 | small = MultiModalKwargsItem.dummy(nbytes=64) |
| 278 | |
| 279 | p0_only = MultiModalProcessorOnlyCache(model_config) |
| 280 | assert p0_only.get_and_update_item((item, []), "big")[0] is item |
| 281 | assert not p0_only.is_cached_item("big") |
| 282 | assert p0_only.get_and_update_item((small, []), "small")[0] is small |
| 283 | assert p0_only.is_cached_item("small") |
| 284 | |
| 285 | p0 = LruKeyReplicatedSenderCache(model_config) |
| 286 | p1 = LruKeyReplicatedReceiverCache(model_config) |
| 287 | assert item is not None |
| 288 | assert p0.get_and_update_item((item, []), "big")[0] is item |
| 289 | assert not p0.is_cached_item("big") |
| 290 | assert p1.get_and_update_item(item, "big") is item |
| 291 | assert "big" not in p1._cache |
| 292 | |
| 293 | |
| 294 | def test_mm_cache_miss_raises_and_recovers(): |
| 295 | """A P0/P1 multimodal cache drift must be recoverable, not a hard crash. |
| 296 | |
| 297 | Reproduces the production failure: the P0 sender cache believes an item is |
| 298 | cached on P1 (so it sends ``data=None``), but the P1 receiver cache does not |
| 299 | have it. The receiver must raise ``MultiModalCacheMissError`` (carrying the |
| 300 | hash) instead of asserting; ``P0.invalidate()`` must drop the stale shadow |
| 301 | entry; and resending the item with data must repopulate both caches. |
| 302 | |
| 303 | Caches are built directly (not via the registry) so the test needs no real |
| 304 | model or network. |
| 305 | """ |
| 306 | model_config = _StubModelConfig(mm_processor_cache_gb=1) |
| 307 | p0 = LruKeyReplicatedSenderCache(model_config) |
| 308 | p1 = LruKeyReplicatedReceiverCache(model_config) |
| 309 | |
| 310 | mm_hash = "image_A" |
| 311 | item = MultiModalKwargsItem.dummy(nbytes=64) |
| 312 | |
| 313 | # Drift: the item reaches P1 with no data (P0 would have sent data=None on a |
| 314 | # stale HIT) but P1 does not have it. Must raise a typed, retryable error |
| 315 | # carrying the hash -- not assert. |
| 316 | with pytest.raises(MultiModalCacheMissError) as exc_info: |
| 317 | p1.get_and_update_item(None, mm_hash) |
| 318 | assert exc_info.value.mm_hashes == [mm_hash] |
| 319 | |
| 320 | # Give P0 a (now stale) shadow entry for the hash. |
| 321 | miss = p0.get_and_update_item((item, []), mm_hash) |
| 322 | assert miss[0] is item # MISS path returns the data to forward to P1 |
| 323 | assert p0.is_cached_item(mm_hash) |
| 324 | # On the next request P0 short-circuits to data=None -- the drift bug when |
| 325 | # P1 lacks the item. |
| 326 | assert item is not None |
| 327 | hit = p0.get_and_update_item((item, []), mm_hash) |
| 328 | assert hit[0] is None |
| 329 | |
| 330 | # Recovery: drop the stale P0 entry so the client's resend-with-data takes |
| 331 | # the MISS path again and repopulates P1. |
| 332 | p0.invalidate(mm_hash) |
| 333 | assert not p0.is_cached_item(mm_hash) |
| 334 | |
| 335 | assert item is not None |
| 336 | resent = p0.get_and_update_item((item, []), mm_hash) |
| 337 | assert resent[0] is item # MISS again -> data is resent |
| 338 | assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it |
| 339 | # A subsequent uuid-only request now succeeds on P1. |
| 340 | assert p1.get_and_update_item(None, mm_hash) == item |
| 341 | |
| 342 | |
| 343 | def test_receiver_cache_replaces_stale_item_when_payload_resent(): |
| 344 | """A P0 miss can resend a different item under the same identity. |
| 345 | |
| 346 | Independent LRU eviction leaves P1 holding the old tensor. Substituting it |
| 347 | for the new payload pairs the request's placeholders with the wrong item |
| 348 | and kills EngineCore. Prefer the fresh payload and keep it for later hits. |
| 349 | """ |
| 350 | model_config = _StubModelConfig(mm_processor_cache_gb=1) |
| 351 | p1 = LruKeyReplicatedReceiverCache(model_config) |
| 352 | small = MultiModalKwargsItem.dummy(nbytes=64) |
| 353 | large = MultiModalKwargsItem.dummy(nbytes=256) |
| 354 | mm_hash = "shared-id" |
| 355 | |
| 356 | assert p1.get_and_update_item(small, mm_hash) is small |
| 357 | assert p1.get_and_update_item(large, mm_hash) is large |
| 358 | assert p1.get_and_update_item(None, mm_hash) is large |
| 359 | |
| 360 | |
| 361 | def test_receiver_cache_features_keep_resent_payload(): |
| 362 | """EngineCore updates features through get_and_update_features.""" |
| 363 | model_config = _StubModelConfig(mm_processor_cache_gb=1) |
| 364 | p1 = LruKeyReplicatedReceiverCache(model_config) |
| 365 | small = MultiModalKwargsItem.dummy(nbytes=64) |
| 366 | large = MultiModalKwargsItem.dummy(nbytes=256) |
| 367 | mm_hash = "shared-id" |
| 368 | |
| 369 | def _feature( |
| 370 | data: MultiModalKwargsItem | None, length: int |
| 371 | ) -> MultiModalFeatureSpec: |
| 372 | return MultiModalFeatureSpec( |
| 373 | data=data, |
| 374 | modality="image", |
| 375 | identifier=mm_hash, |
| 376 | mm_position=PlaceholderRange(offset=0, length=length), |
| 377 | mm_hash=mm_hash, |
| 378 | ) |
| 379 | |
| 380 | seeded = p1.get_and_update_features([_feature(small, length=4)]) |
| 381 | assert seeded[0].data is small |
| 382 | |
| 383 | updated = p1.get_and_update_features([_feature(large, length=2048)]) |
| 384 | assert updated[0].data is large |
| 385 | assert updated[0].mm_position.length == 2048 |
| 386 | |
| 387 | |
| 388 | def test_mm_cache_miss_batches_all_drifted_hashes(): |
| 389 | """All hashes drifted within one request must surface in a single error. |
| 390 | |
| 391 | Otherwise a request with k drifted items needs k client retries (each resend |
| 392 | un-shadows only the one reported hash). get_and_update_features collects every |
| 393 | miss and raises once, so a single retry recovers the whole request -- while the |
| 394 | non-drifted items in the same request are still ingested. |
| 395 | """ |
| 396 | model_config = _StubModelConfig(mm_processor_cache_gb=1) |
| 397 | p1 = LruKeyReplicatedReceiverCache(model_config) |
| 398 | item = MultiModalKwargsItem.dummy(nbytes=64) |
| 399 | |
| 400 | def _feature( |
| 401 | mm_hash: str, data: MultiModalKwargsItem | None |
| 402 | ) -> MultiModalFeatureSpec: |
| 403 | return MultiModalFeatureSpec( |
| 404 | data=data, |
| 405 | modality="image", |
| 406 | identifier=mm_hash, |
| 407 | mm_position=PlaceholderRange(offset=0, length=1), |
| 408 | mm_hash=mm_hash, |
| 409 | ) |
| 410 | |
| 411 | # A and C carry data; B, D, E arrive data=None (stale P0 shadow HITs) but P1 |
| 412 | # has none of them -- all three must be reported together, in order. |
| 413 | features = [ |
| 414 | _feature("A", item), |
| 415 | _feature("B", None), |
| 416 | _feature("C", item), |
| 417 | _feature("D", None), |
| 418 | _feature("E", None), |
| 419 | ] |
| 420 | with pytest.raises(MultiModalCacheMissError) as exc_info: |
| 421 | p1.get_and_update_features(features) |
| 422 | assert exc_info.value.mm_hashes == ["B", "D", "E"] |
| 423 | # The non-drifted items were still ingested into P1 before the raise. |
| 424 | assert "A" in p1._cache and "C" in p1._cache |
| 425 | |
| 426 | |
| 427 | def test_shm_receiver_handles_prefix_covered_items(monkeypatch): |
| 428 | """Regression test for the EngineCore crash with prefix caching + SHM cache. |
| 429 | |
| 430 | On a repeated request, prefix caching can fully cover an item's |
| 431 | placeholder range, so strip_covered_mm_data() drops its payload: uncached |
| 432 | items arrive at the worker with data=None, while SHM-cached items keep |
| 433 | their address descriptor. Before the fix the SHM receiver asserted on the |
| 434 | None item, terminating EngineCore on the second identical request |
| 435 | (vllm-project/vllm#54994). The address item must still be resolved and |
| 436 | acknowledged so the object stays reclaimable. |
| 437 | """ |
| 438 | monkeypatch.setenv( |
| 439 | "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "test_shm_prefix_covered_items" |
| 440 | ) |
| 441 | vllm_config = SimpleNamespace( |
| 442 | parallel_config=SimpleNamespace(world_size=1), |
| 443 | model_config=_StubModelConfig(mm_processor_cache_gb=4 * MiB_bytes / GiB_bytes), |
| 444 | ) |
| 445 | p0 = ShmObjectStoreSenderCache(vllm_config) |
| 446 | p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) |
| 447 | |
| 448 | def _feature( |
| 449 | mm_hash: str, data: MultiModalKwargsItem | None |
| 450 | ) -> MultiModalFeatureSpec: |
| 451 | return MultiModalFeatureSpec( |
| 452 | data=data, |
| 453 | modality="image", |
| 454 | identifier=mm_hash, |
| 455 | mm_position=PlaceholderRange(offset=0, length=100), |
| 456 | mm_hash=mm_hash, |
| 457 | ) |
| 458 | |
| 459 | mm_hash = "image_A" |
| 460 | item = MultiModalKwargsItem.dummy(nbytes=1024) |
| 461 | |
| 462 | try: |
| 463 | # Request 1 (sender miss): the payload is put into SHM; the worker |
| 464 | # receives an address descriptor and resolves it. |
| 465 | p0.touch_sender_cache_item(mm_hash) |
| 466 | address_item, _ = p0.get_and_update_item((item, []), mm_hash) |
| 467 | first = _feature(mm_hash, address_item) |
| 468 | p1.get_and_update_features([first]) |
| 469 | assert first.data is not None |
| 470 | assert torch.equal(first.data["dummy"].data, item["dummy"].data) |
| 471 | |
| 472 | # Request 2 (identical, fully prefix-covered): the sender hit takes |
| 473 | # writer references (touch + get_cached) and returns the address |
| 474 | # descriptor; the scheduler strips the covered uncached payload to |
| 475 | # data=None. |
| 476 | p0.touch_sender_cache_item(mm_hash) |
| 477 | address_item_2, _ = p0.get_and_update_item(None, mm_hash) |
| 478 | covered_uncached = _feature("image_B", None) |
| 479 | covered_cached = _feature(mm_hash, address_item_2) |
| 480 | |
| 481 | # Pre-fix this asserted in touch_receiver_cache_item and terminated |
| 482 | # EngineCore. |
| 483 | p1.get_and_update_features([covered_uncached, covered_cached]) |
| 484 | |
| 485 | assert covered_uncached.data is None |
| 486 | # The address item is resolved to the cached payload. |
| 487 | assert covered_cached.data is not None |
| 488 | assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data) |
| 489 | |
| 490 | # The hit's writer references were acknowledged by the worker, so the |
| 491 | # object remains reclaimable instead of permanently protected. |
| 492 | p0._shm_cache.free_unused() |
| 493 | assert not p0._shm_cache.is_cached(mm_hash) |
| 494 | finally: |
| 495 | p1._shm_cache.close() |
| 496 | p0.close() |
| 497 | |
| 498 | |
| 499 | def _run_test_cache_eviction_lru( |
| 500 | p0_cache: LruKeyReplicatedSenderCache, |
| 501 | p1_cache: BaseMultiModalReceiverCache, |
| 502 | base_item_size: int, |
| 503 | ): |
| 504 | request1_hashes = [ |
| 505 | "image_A", |
| 506 | "image_B", |
| 507 | "image_C", |
| 508 | ] |
| 509 | request1_items = { |
| 510 | h: MultiModalKwargsItem.dummy(nbytes=2 * base_item_size) |
| 511 | for h in request1_hashes |
| 512 | } |
| 513 | |
| 514 | request2_hashes = ["image_D", "image_E", "image_A", "image_C"] |
| 515 | request2_items = { |
| 516 | h: MultiModalKwargsItem.dummy(nbytes=1 * base_item_size) |
| 517 | for h in request2_hashes |
| 518 | } |
| 519 | |
| 520 | ########################## |
| 521 | # STEP 1: Request 1 send |
| 522 | ########################## |
| 523 | sender_is_cached_item_req1 = p0_cache.is_cached(request1_hashes) |
| 524 | # Cache is empty |
| 525 | assert sender_is_cached_item_req1 == [False, False, False] |
| 526 | |
| 527 | # Touch all mm hash for P0 Cache before process |
| 528 | for mm_hash in request1_hashes: |
| 529 | p0_cache.touch_sender_cache_item(mm_hash) |
| 530 | |
| 531 | ########################### |
| 532 | # Process request 1 for P0 Cache |
| 533 | ########################### |
| 534 | item_tuple: MultiModalProcessorCacheInItem |
| 535 | for i, h in enumerate(request1_hashes): |
| 536 | # Use precomputed cache state |
| 537 | is_cached = sender_is_cached_item_req1[i] |
| 538 | item_tuple = (request1_items[h], []) if not is_cached else None |
| 539 | print(f"Request 1: key={h} | cached={is_cached}") |
| 540 | |
| 541 | p0_cache.get_and_update_item(item_tuple, h) |
| 542 | |
| 543 | ########################### |
| 544 | # Process request 1 for P1 Cache |
| 545 | ########################### |
| 546 | # Touch all mm hash for P1 Cache before process |
| 547 | for mm_hash in request1_hashes: |
| 548 | p1_cache.touch_receiver_cache_item(mm_hash) |
| 549 | |
| 550 | for h in request1_hashes: |
| 551 | p1_cache.get_and_update_item(request1_items[h], h) |
| 552 | |
| 553 | expected_hashes = ["image_A", "image_B", "image_C"] |
| 554 | assert list(p0_cache._cache.order) == expected_hashes |
| 555 | |
| 556 | ########################## |
| 557 | # STEP 2: Request 2 send |
| 558 | ########################## |
| 559 | sender_is_cached_item_req2 = p0_cache.is_cached(request2_hashes) |
| 560 | assert sender_is_cached_item_req2 == [False, False, True, True] |
| 561 | |
| 562 | # Touch all mm hash for P0 Cache before process |
| 563 | for mm_hash in request2_hashes: |
| 564 | p0_cache.touch_sender_cache_item(mm_hash) |
| 565 | |
| 566 | ########################### |
| 567 | # Process request 2 for P0 Cache |
| 568 | ########################### |
| 569 | for i, h in enumerate(request2_hashes): |
| 570 | # Use precomputed cache state again |
| 571 | is_cached = sender_is_cached_item_req2[i] |
| 572 | item_tuple = (request2_items[h], []) if not is_cached else None |
| 573 | print(f"Request 2: key={h} | cached={is_cached}") |
| 574 | |
| 575 | p0_cache.get_and_update_item(item_tuple, h) |
| 576 | |
| 577 | ########################### |
| 578 | # Process request 2 for P1 Cache |
| 579 | ########################### |
| 580 | |
| 581 | # Touch all mm hash for P1 Cache before process |
| 582 | for mm_hash in request2_hashes: |
| 583 | p1_cache.touch_receiver_cache_item(mm_hash) |
| 584 | |
| 585 | for h in request2_hashes: |
| 586 | p1_cache.get_and_update_item(request2_items[h], h) |
| 587 | |
| 588 | expected_hashes = ["image_D", "image_E", "image_A", "image_C"] |
| 589 | assert list(p0_cache._cache.order) == expected_hashes |
| 590 | |
| 591 | |
| 592 | def test_cache_eviction_lru_cache(): |
| 593 | model_config = ModelConfig( |
| 594 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 595 | mm_processor_cache_gb=6 / GiB_bytes, |
| 596 | ) |
| 597 | sender_cache = LruKeyReplicatedSenderCache(model_config) |
| 598 | receiver_cache = LruKeyReplicatedReceiverCache(model_config) |
| 599 | |
| 600 | _run_test_cache_eviction_lru(sender_cache, receiver_cache, base_item_size=1) |
| 601 | |
| 602 | |
| 603 | # This test verifies shared-memory cache eviction behavior across processor (p0) |
| 604 | # and receiver (p1) caches. |
| 605 | # Flow summary: |
| 606 | # 1. Request 1 adds images A, B, C — completely filling the cache. |
| 607 | # 2. Request 2 tries to add image_G and image_A, but image_G cannot be added because |
| 608 | # cache is full and A is protected from eviction — cache remains unchanged. |
| 609 | # 3. Request 3 adds image_G, image_H, image_I and image_B |
| 610 | # this time, image_A is evicted, freeing 5MB space |
| 611 | # and image_G, image_H successfully fits, |
| 612 | # image_B is protected from eviction then image_i cannot be added. |
| 613 | # This proving normal eviction and reuse behavior. |
| 614 | def _run_test_cache_eviction_shm( |
| 615 | p0_cache: ShmObjectStoreSenderCache, |
| 616 | p1_cache: BaseMultiModalReceiverCache, |
| 617 | base_item_size: int, |
| 618 | ): |
| 619 | request1_hashes = ["image_A", "image_B", "image_C"] |
| 620 | request1_items = { |
| 621 | h: MultiModalKwargsItem.dummy(5 * base_item_size) for h in request1_hashes |
| 622 | } |
| 623 | request1_items_p0_result = [] |
| 624 | |
| 625 | request2_hashes = ["image_G", "image_A"] |
| 626 | request2_items = { |
| 627 | h: MultiModalKwargsItem.dummy( |
| 628 | (5 if h in request1_hashes else 2) * base_item_size |
| 629 | ) |
| 630 | for h in request2_hashes |
| 631 | } |
| 632 | request2_items_p0_result = [] |
| 633 | |
| 634 | request3_hashes = ["image_G", "image_H", "image_I", "image_B"] |
| 635 | request3_items = { |
| 636 | h: MultiModalKwargsItem.dummy( |
| 637 | (5 if h in request1_hashes else 2) * base_item_size |
| 638 | ) |
| 639 | for h in request3_hashes |
| 640 | } |
| 641 | request3_items_p0_result = [] |
| 642 | |
| 643 | ########################## |
| 644 | # STEP 1: Request 1 send |
| 645 | # This will fill up the cache |
| 646 | ########################## |
| 647 | sender_is_cached_item_req1 = p0_cache.is_cached(request1_hashes) |
| 648 | # Cache is empty |
| 649 | assert sender_is_cached_item_req1 == [False, False, False] |
| 650 | |
| 651 | # Touch all mm hash for P0 Cache before process |
| 652 | for mm_hash in request1_hashes: |
| 653 | p0_cache.touch_sender_cache_item(mm_hash) |
| 654 | |
| 655 | ########################### |
| 656 | # Process request 1 for P0 Cache |
| 657 | ########################### |
| 658 | item_tuple: MultiModalProcessorCacheInItem |
| 659 | for i, h in enumerate(request1_hashes): |
| 660 | # Use precomputed cache state |
| 661 | is_cached = sender_is_cached_item_req1[i] |
| 662 | item_tuple = (request1_items[h], []) if not is_cached else None |
| 663 | print(f"Request 1: key={h} | cached={is_cached}") |
| 664 | |
| 665 | p0_result = p0_cache.get_and_update_item(item_tuple, h) |
| 666 | # Only get mm item, ignore prompt update result |
| 667 | request1_items_p0_result.append(p0_result[0]) |
| 668 | |
| 669 | ########################### |
| 670 | # Process request 1 for P1 Cache |
| 671 | ########################### |
| 672 | # Touch all mm hash for P1 Cache before process |
| 673 | for mm_hash, mm_item in zip(request1_hashes, request1_items_p0_result): |
| 674 | p1_cache.touch_receiver_cache_item(mm_hash, mm_item) |
| 675 | |
| 676 | for mm_hash, mm_item in zip(request1_hashes, request1_items_p0_result): |
| 677 | p1_cache.get_and_update_item(mm_item, mm_hash) |
| 678 | |
| 679 | expected_hashes = ["image_A", "image_B", "image_C"] |
| 680 | assert list(p0_cache._shm_cache.key_index.keys()) == expected_hashes |
| 681 | |
| 682 | ########################## |
| 683 | # STEP 2: Request 2 send |
| 684 | # There is no eviction because image_A is protected |
| 685 | # No new item can add to cache |
| 686 | ########################## |
| 687 | sender_is_cached_item_req2 = p0_cache.is_cached(request2_hashes) |
| 688 | assert sender_is_cached_item_req2 == [False, True] |
| 689 | |
| 690 | # Touch all mm hash for P0 Cache before process |
| 691 | for mm_hash in request2_hashes: |
| 692 | p0_cache.touch_sender_cache_item(mm_hash) |
| 693 | |
| 694 | ########################### |
| 695 | # Process request 2 for P0 Cache |
| 696 | ########################### |
| 697 | for i, h in enumerate(request2_hashes): |
| 698 | # Use precomputed cache state again |
| 699 | is_cached = sender_is_cached_item_req2[i] |
| 700 | item_tuple = (request2_items[h], []) if not is_cached else None |
| 701 | print(f"Request 2: key={h} | cached={is_cached}") |
| 702 | |
| 703 | p0_result = p0_cache.get_and_update_item(item_tuple, h) |
| 704 | # Only get mm item, ignore prompt update result |
| 705 | request2_items_p0_result.append(p0_result[0]) |
| 706 | |
| 707 | # image_A cannot be evict then |
| 708 | # image_G will fail to allocate anyway and image_A still in cache |
| 709 | assert p0_cache.is_cached(request2_hashes) == [False, True] |
| 710 | |
| 711 | ########################### |
| 712 | # Process request 2 for P1 Cache |
| 713 | ########################### |
| 714 | |
| 715 | # Touch all mm hash for P1 Cache before process |
| 716 | for mm_hash, mm_item in zip(request2_hashes, request2_items_p0_result): |
| 717 | p1_cache.touch_receiver_cache_item(mm_hash, mm_item) |
| 718 | |
| 719 | for mm_hash, mm_item in zip(request2_hashes, request2_items_p0_result): |
| 720 | p1_cache.get_and_update_item(mm_item, mm_hash) |
| 721 | |
| 722 | # Prove that cache state is unchanged |
| 723 | expected_hashes = ["image_A", "image_B", "image_C"] |
| 724 | assert list(p0_cache._shm_cache.key_index.keys()) == expected_hashes |
| 725 | |
| 726 | ########################## |
| 727 | # STEP 3: Request 3 send |
| 728 | ########################## |
| 729 | ##### Prove that cache eviction work normally |
| 730 | sender_is_cached_item_req3 = p0_cache.is_cached(request3_hashes) |
| 731 | assert sender_is_cached_item_req3 == [False, False, False, True] |
| 732 | |
| 733 | # Touch all mm hash for P0 Cache before process |
| 734 | for mm_hash in request3_hashes: |
| 735 | p0_cache.touch_sender_cache_item(mm_hash) |
| 736 | |
| 737 | ########################### |
| 738 | # Process request 3 for P0 Cache |
| 739 | ########################### |
| 740 | for i, h in enumerate(request3_hashes): |
| 741 | # Use precomputed cache state again |
| 742 | is_cached = sender_is_cached_item_req3[i] |
| 743 | item_tuple = (request3_items[h], []) if not is_cached else None |
| 744 | print(f"Request 3: key={h} | cached={is_cached}") |
| 745 | p0_result = p0_cache.get_and_update_item(item_tuple, h) |
| 746 | # Only get mm item, ignore prompt update result |
| 747 | request3_items_p0_result.append(p0_result[0]) |
| 748 | |
| 749 | # image_A got evict and image_G add to cache |
| 750 | # image_B is still protected |
| 751 | # image_G, image_H fit but image_I cannot fit |
| 752 | assert p0_cache.is_cached(request3_hashes) == [True, True, False, True] |
| 753 | |
| 754 | ########################### |
| 755 | # Process request 3 for P1 Cache |
| 756 | ########################### |
| 757 | |
| 758 | # Touch all mm hash for P1 Cache before process |
| 759 | for mm_hash, mm_item in zip(request3_hashes, request3_items_p0_result): |
| 760 | p1_cache.touch_receiver_cache_item(mm_hash, mm_item) |
| 761 | |
| 762 | for mm_hash, mm_item in zip(request3_hashes, request3_items_p0_result): |
| 763 | p1_cache.get_and_update_item(mm_item, mm_hash) |
| 764 | |
| 765 | expected_hashes = ["image_B", "image_C", "image_G", "image_H"] |
| 766 | assert list(p0_cache._shm_cache.key_index.keys()) == expected_hashes |
| 767 | |
| 768 | |
| 769 | def test_cache_eviction_shm_cache(): |
| 770 | vllm_config = VllmConfig( |
| 771 | model_config=ModelConfig( |
| 772 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 773 | mm_processor_cache_type="shm", |
| 774 | mm_shm_cache_max_object_size_mb=6, |
| 775 | mm_processor_cache_gb=15.2 * MiB_bytes / GiB_bytes, |
| 776 | ), |
| 777 | ) |
| 778 | sender_cache = ShmObjectStoreSenderCache(vllm_config) |
| 779 | receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) |
| 780 | |
| 781 | _run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes) |
| 782 | |
| 783 | |
| 784 | def test_shm_receiver_requires_sender_handle_for_current_cache_key(): |
| 785 | vllm_config = VllmConfig( |
| 786 | model_config=ModelConfig( |
| 787 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 788 | mm_processor_cache_type="shm", |
| 789 | mm_shm_cache_max_object_size_mb=1, |
| 790 | mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, |
| 791 | ), |
| 792 | ) |
| 793 | sender_cache = ShmObjectStoreSenderCache(vllm_config) |
| 794 | receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) |
| 795 | mm_hash = "image_A" |
| 796 | item = MultiModalKwargsItem.dummy(1024) |
| 797 | |
| 798 | try: |
| 799 | sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) |
| 800 | assert sender_item is not None |
| 801 | |
| 802 | valid_feature = MultiModalFeatureSpec( |
| 803 | data=sender_item, |
| 804 | modality="image", |
| 805 | identifier=mm_hash, |
| 806 | mm_position=PlaceholderRange(offset=0, length=1), |
| 807 | mm_hash=mm_hash, |
| 808 | ) |
| 809 | receiver_cache.get_and_update_features([valid_feature]) |
| 810 | assert valid_feature.data == item |
| 811 | |
| 812 | forged_item = MultiModalKwargsItem( |
| 813 | { |
| 814 | "address": sender_item["address"], |
| 815 | "monotonic_id": sender_item["monotonic_id"], |
| 816 | } |
| 817 | ) |
| 818 | forged_feature = MultiModalFeatureSpec( |
| 819 | data=forged_item, |
| 820 | modality="image", |
| 821 | identifier=mm_hash, |
| 822 | mm_position=PlaceholderRange(offset=0, length=1), |
| 823 | mm_hash=mm_hash, |
| 824 | ) |
| 825 | with pytest.raises(ValueError, match="SHM handle signature"): |
| 826 | receiver_cache.get_and_update_features([forged_feature]) |
| 827 | |
| 828 | wrong_key = "image_B" |
| 829 | wrong_key_feature = MultiModalFeatureSpec( |
| 830 | data=sender_item, |
| 831 | modality="image", |
| 832 | identifier=wrong_key, |
| 833 | mm_position=PlaceholderRange(offset=0, length=1), |
| 834 | mm_hash=wrong_key, |
| 835 | ) |
| 836 | with pytest.raises(ValueError, match="SHM handle signature"): |
| 837 | receiver_cache.get_and_update_features([wrong_key_feature]) |
| 838 | finally: |
| 839 | receiver_cache._shm_cache.close() |
| 840 | sender_cache.close() |
| 841 | |
| 842 | |
| 843 | def test_shm_sender_handle_survives_engine_msgpack_round_trip(): |
| 844 | vllm_config = VllmConfig( |
| 845 | model_config=ModelConfig( |
| 846 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 847 | mm_processor_cache_type="shm", |
| 848 | mm_shm_cache_max_object_size_mb=1, |
| 849 | mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, |
| 850 | ), |
| 851 | ) |
| 852 | sender_cache = ShmObjectStoreSenderCache(vllm_config) |
| 853 | receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) |
| 854 | mm_hash = "image_A" |
| 855 | item = MultiModalKwargsItem.dummy(1024) |
| 856 | |
| 857 | try: |
| 858 | sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) |
| 859 | encoded = MsgpackEncoder().encode(sender_item) |
| 860 | decoded = MsgpackDecoder(MultiModalKwargsItem).decode(encoded) |
| 861 | feature = MultiModalFeatureSpec( |
| 862 | data=decoded, |
| 863 | modality="image", |
| 864 | identifier=mm_hash, |
| 865 | mm_position=PlaceholderRange(offset=0, length=1), |
| 866 | mm_hash=mm_hash, |
| 867 | ) |
| 868 | |
| 869 | receiver_cache.get_and_update_features([feature]) |
| 870 | |
| 871 | assert feature.data == item |
| 872 | finally: |
| 873 | receiver_cache._shm_cache.close() |
| 874 | sender_cache.close() |
| 875 | |
| 876 | |
| 877 | def _small_shm_vllm_config() -> VllmConfig: |
| 878 | return VllmConfig( |
| 879 | model_config=ModelConfig( |
| 880 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 881 | mm_processor_cache_type="shm", |
| 882 | mm_shm_cache_max_object_size_mb=1, |
| 883 | mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, |
| 884 | ), |
| 885 | ) |
| 886 | |
| 887 | |
| 888 | def _with_elem(item: MultiModalKwargsItem, key: str, data) -> MultiModalKwargsItem: |
| 889 | elems = dict(item) |
| 890 | elems[key] = MultiModalFieldElem(data=data, field=MultiModalBatchedField()) |
| 891 | return MultiModalKwargsItem(elems) |
| 892 | |
| 893 | |
| 894 | def test_shm_sender_validates_client_supplied_handles(): |
| 895 | sender_cache = ShmObjectStoreSenderCache(_small_shm_vllm_config()) |
| 896 | mm_hash = "image_A" |
| 897 | |
| 898 | try: |
| 899 | handle, _ = sender_cache.get_and_update_item( |
| 900 | (MultiModalKwargsItem.dummy(1024), []), mm_hash |
| 901 | ) |
| 902 | assert handle is not None |
| 903 | |
| 904 | sender_cache.validate_input_item(handle, mm_hash) |
| 905 | sender_cache.validate_input_item(MultiModalKwargsItem.dummy(16), mm_hash) |
| 906 | |
| 907 | signature = cast(list[int], handle["signature"].data) |
| 908 | forged = [ |
| 909 | _with_elem(handle, "signature", [signature[0] ^ 1, *signature[1:]]), |
| 910 | _with_elem(handle, "signature", [signature]), |
| 911 | _with_elem(handle, "address", True), |
| 912 | _with_elem(handle, "address", -1), |
| 913 | _with_elem(handle, "monotonic_id", 2**32), |
| 914 | MultiModalKwargsItem( |
| 915 | {"address": handle["address"], "monotonic_id": handle["monotonic_id"]} |
| 916 | ), |
| 917 | ] |
| 918 | for item in forged: |
| 919 | with pytest.raises(ValueError, match="SHM handle signature"): |
| 920 | sender_cache.validate_input_item(item, mm_hash) |
| 921 | with pytest.raises(ValueError, match="SHM handle signature"): |
| 922 | sender_cache.validate_input_item(handle, "image_B") |
| 923 | |
| 924 | sender_cache.clear_cache() |
| 925 | with pytest.raises(ValueError, match="SHM handle signature"): |
| 926 | sender_cache.validate_input_item(handle, mm_hash) |
| 927 | finally: |
| 928 | sender_cache.close() |
| 929 | |
| 930 | |
| 931 | def test_shm_receiver_rejects_out_of_range_handle_fields(): |
| 932 | vllm_config = _small_shm_vllm_config() |
| 933 | sender_cache = ShmObjectStoreSenderCache(vllm_config) |
| 934 | receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) |
| 935 | mm_hash = "image_A" |
| 936 | |
| 937 | try: |
| 938 | handle, _ = sender_cache.get_and_update_item( |
| 939 | (MultiModalKwargsItem.dummy(1024), []), mm_hash |
| 940 | ) |
| 941 | assert handle is not None |
| 942 | |
| 943 | for key, value in [ |
| 944 | ("address", -1), |
| 945 | ("address", 2**64), |
| 946 | ("monotonic_id", -1), |
| 947 | ("monotonic_id", 2**32), |
| 948 | ]: |
| 949 | with pytest.raises(ValueError, match="SHM handle signature"): |
| 950 | receiver_cache.get_and_update_item( |
| 951 | _with_elem(handle, key, value), mm_hash |
| 952 | ) |
| 953 | finally: |
| 954 | receiver_cache._shm_cache.close() |
| 955 | sender_cache.close() |
| 956 | |
| 957 | |
| 958 | def test_shm_handle_issued_before_cache_clear(): |
| 959 | vllm_config = _small_shm_vllm_config() |
| 960 | sender_cache = ShmObjectStoreSenderCache(vllm_config) |
| 961 | receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) |
| 962 | mm_hash = "image_A" |
| 963 | item = MultiModalKwargsItem.dummy(1024) |
| 964 | |
| 965 | def feature(data: MultiModalKwargsItem) -> MultiModalFeatureSpec: |
| 966 | return MultiModalFeatureSpec( |
| 967 | data=data, |
| 968 | modality="image", |
| 969 | identifier=mm_hash, |
| 970 | mm_position=PlaceholderRange(offset=0, length=1), |
| 971 | mm_hash=mm_hash, |
| 972 | ) |
| 973 | |
| 974 | try: |
| 975 | stale_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) |
| 976 | assert stale_item is not None |
| 977 | |
| 978 | sender_cache.clear_cache() |
| 979 | |
| 980 | # A request still draining after the clear resolves to its own data. |
| 981 | draining = feature(stale_item) |
| 982 | receiver_cache.get_and_update_features([draining]) |
| 983 | assert draining.data == item |
| 984 | |
| 985 | sender_cache.get_and_update_item( |
| 986 | (MultiModalKwargsItem.dummy(2048), []), |
| 987 | mm_hash, |
| 988 | ) |
| 989 | |
| 990 | with pytest.raises(ValueError, match="modified or is invalid"): |
| 991 | receiver_cache.get_and_update_features([feature(stale_item)]) |
| 992 | finally: |
| 993 | receiver_cache._shm_cache.close() |
| 994 | sender_cache.close() |
| 995 | |
| 996 | |
| 997 | def test_processor_cache_shared_across_loras(): |
| 998 | """Test that processor cache uses mm_hash to share data across LoRAs.""" |
| 999 | model_config = ModelConfig( |
| 1000 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 1001 | mm_processor_cache_gb=1, |
| 1002 | ) |
| 1003 | receiver_cache = LruKeyReplicatedReceiverCache(model_config) |
| 1004 | |
| 1005 | base_mm_hash = "image_hash_abc123" |
| 1006 | lora_a_identifier = f"12345:{base_mm_hash}" |
| 1007 | lora_b_identifier = f"67890:{base_mm_hash}" |
| 1008 | |
| 1009 | item_data = MultiModalKwargsItem.dummy(1024) |
| 1010 | |
| 1011 | feature_lora_a = MultiModalFeatureSpec( |
| 1012 | data=item_data, |
| 1013 | modality="image", |
| 1014 | identifier=lora_a_identifier, |
| 1015 | mm_position=PlaceholderRange(offset=0, length=100), |
| 1016 | mm_hash=base_mm_hash, |
| 1017 | ) |
| 1018 | |
| 1019 | receiver_cache.get_and_update_features([feature_lora_a]) |
| 1020 | assert base_mm_hash in receiver_cache._cache |
| 1021 | |
| 1022 | feature_lora_b = MultiModalFeatureSpec( |
| 1023 | data=None, |
| 1024 | modality="image", |
| 1025 | identifier=lora_b_identifier, |
| 1026 | mm_position=PlaceholderRange(offset=0, length=100), |
| 1027 | mm_hash=base_mm_hash, |
| 1028 | ) |
| 1029 | |
| 1030 | receiver_cache.get_and_update_features([feature_lora_b]) |
| 1031 | assert feature_lora_b.data == item_data |
| 1032 | |
| 1033 | |
| 1034 | @pytest.mark.parametrize("use_async", [False, True]) |
| 1035 | @pytest.mark.parametrize( |
| 1036 | "release_error", |
| 1037 | [ |
| 1038 | None, |
| 1039 | "requires a completed pause first", |
| 1040 | "requires all executor memory to be resident", |
| 1041 | ], |
| 1042 | ids=["released", "not-paused", "nonresident-memory"], |
| 1043 | ) |
| 1044 | @pytest.mark.asyncio |
| 1045 | async def test_release_kv_cache_resends_mm_payload(use_async, release_error): |
| 1046 | """Release must not leave a sender hit pointing at a cleared receiver.""" |
| 1047 | ctx = build_model_context( |
| 1048 | "llava-hf/llava-v1.6-mistral-7b-hf", |
| 1049 | mm_processor_kwargs=None, |
| 1050 | limit_mm_per_prompt={"image": 1}, |
| 1051 | mm_processor_cache_gb=1, |
| 1052 | ) |
| 1053 | model_config = ctx.model_config |
| 1054 | |
| 1055 | sender = LruKeyReplicatedSenderCache(model_config) |
| 1056 | receiver = LruKeyReplicatedReceiverCache(model_config) |
| 1057 | item = _dummy_item({"pixel_values": 16}) |
| 1058 | mm_hash = "image_A" |
| 1059 | payload, _ = sender.get_and_update_item((item, []), mm_hash) |
| 1060 | assert receiver.get_and_update_item(payload, mm_hash) is item |
| 1061 | assert sender.get_and_update_item(None, mm_hash)[0] is None |
| 1062 | |
| 1063 | renderer = renderer_from_config(VllmConfig(model_config=model_config)) |
| 1064 | renderer._mm_processor_cache = sender |
| 1065 | |
| 1066 | def release(): |
| 1067 | if release_error: |
| 1068 | raise RuntimeError(release_error) |
| 1069 | receiver.clear_cache() |
| 1070 | |
| 1071 | engine = SimpleNamespace( |
| 1072 | renderer=renderer, |
| 1073 | engine_core=SimpleNamespace( |
| 1074 | release_kv_cache_memory=release, |
| 1075 | release_kv_cache_memory_async=make_async( |
| 1076 | release, |
| 1077 | executor=renderer._executor, |
| 1078 | ), |
| 1079 | ), |
| 1080 | logger_manager=Mock(), |
| 1081 | ) |
| 1082 | |
| 1083 | async def call_release(): |
| 1084 | if use_async: |
| 1085 | await AsyncLLM.release_kv_cache_memory(engine) |
| 1086 | else: |
| 1087 | LLMEngine.release_kv_cache_memory(engine) |
| 1088 | |
| 1089 | if release_error: |
| 1090 | with pytest.raises(RuntimeError, match=release_error): |
| 1091 | await call_release() |
| 1092 | engine.logger_manager.record_sleep_state.assert_not_called() |
| 1093 | else: |
| 1094 | await call_release() |
| 1095 | engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0) |
| 1096 | |
| 1097 | payload, _ = sender.get_and_update_item((item, []), mm_hash) |
| 1098 | assert payload is item |
| 1099 | assert receiver.get_and_update_item(payload, mm_hash) is item |
| 1100 | |
| 1101 | |
| 1102 | _SLEEP_VISION_PROMPT = ( |
| 1103 | "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" |
| 1104 | "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" |
| 1105 | "What is in the image?<|im_end|>\n" |
| 1106 | "<|im_start|>assistant\n" |
| 1107 | ) |
| 1108 | |
| 1109 | |
| 1110 | @create_new_process_for_each_test() |
| 1111 | @pytest.mark.skipif( |
| 1112 | not torch.cuda.is_available(), |
| 1113 | reason="sleep mode regression requires a CUDA GPU", |
| 1114 | ) |
| 1115 | def test_sleep_wake_preserves_mm_cache_consistency(): |
| 1116 | """Regression for vllm-project/vllm#42995.""" |
| 1117 | from vllm import LLM, SamplingParams |
| 1118 | from vllm.assets.image import ImageAsset |
| 1119 | from vllm.inputs import TextPrompt |
| 1120 | |
| 1121 | image = ImageAsset("stop_sign").pil_image |
| 1122 | prompt: TextPrompt = { |
| 1123 | "prompt": _SLEEP_VISION_PROMPT, |
| 1124 | "multi_modal_data": {"image": image}, |
| 1125 | } |
| 1126 | sampling_params = SamplingParams(temperature=0, max_tokens=8) |
| 1127 | |
| 1128 | llm = LLM( |
| 1129 | model="Qwen/Qwen2-VL-2B-Instruct", |
| 1130 | enable_sleep_mode=True, |
| 1131 | enforce_eager=True, |
| 1132 | gpu_memory_utilization=0.5, |
| 1133 | max_model_len=2048, |
| 1134 | ) |
| 1135 | |
| 1136 | llm.generate([prompt], sampling_params) |
| 1137 | llm.sleep(level=1) |
| 1138 | llm.wake_up() |
| 1139 | output2 = llm.generate([prompt], sampling_params) |
| 1140 | assert output2[0].outputs[0].text |