[Bugfix][Multimodal] Handle prefix-covered items in SHM worker cache (#54994)
Signed-off-by: waizuichougou <2082431897@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
waizuichougou committed
Sep 3, 2026 at 14:39 UTC
3f1af35e012a283a58caf17216c108d8fca9e6c5
4 files changed
+115
-4
tests/multimodal/test_cache.py
+71
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import multiprocessing as mp
4
+from types import SimpleNamespace
5
6
import numpy as np
7
import pytest
@@ -362,6 +363,76 @@ def test_mm_cache_miss_batches_all_drifted_hashes():
363
assert "A" in p1._cache and "C" in p1._cache
364
365
366
+def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
367
+ """Regression test for the EngineCore crash with prefix caching + SHM cache.
368
+
369
+ On a repeated request, prefix caching can fully cover an item's
370
+ placeholder range, so strip_covered_mm_data() drops its payload: uncached
371
+ items arrive at the worker with data=None, while SHM-cached items keep
372
+ their address descriptor. Before the fix the SHM receiver asserted on the
373
+ None item, terminating EngineCore on the second identical request
374
+ (vllm-project/vllm#54994). The address item must still be resolved and
375
+ acknowledged so the object stays reclaimable.
376
+ """
377
+ monkeypatch.setenv(
378
+ "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "test_shm_prefix_covered_items"
379
+ )
380
+ vllm_config = SimpleNamespace(
381
+ parallel_config=SimpleNamespace(world_size=1),
382
+ model_config=_StubModelConfig(mm_processor_cache_gb=4 * MiB_bytes / GiB_bytes),
383
+ )
384
+ p0 = ShmObjectStoreSenderCache(vllm_config) # type: ignore[arg-type]
385
+ p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) # type: ignore[arg-type]
386
+
387
+ def _feature(
388
+ mm_hash: str, data: MultiModalKwargsItem | None
389
+ ) -> MultiModalFeatureSpec:
390
+ return MultiModalFeatureSpec(
391
+ data=data,
392
+ modality="image",
393
+ identifier=mm_hash,
394
+ mm_position=PlaceholderRange(offset=0, length=100),
395
+ mm_hash=mm_hash,
396
+ )
397
+
398
+ mm_hash = "image_A"
399
+ item = MultiModalKwargsItem.dummy(nbytes=1024)
400
+
401
+ try:
402
+ # Request 1 (sender miss): the payload is put into SHM; the worker
403
+ # receives an address descriptor and resolves it.
404
+ p0.touch_sender_cache_item(mm_hash)
405
+ address_item, _ = p0.get_and_update_item((item, []), mm_hash)
406
+ first = _feature(mm_hash, address_item)
407
+ p1.get_and_update_features([first])
408
+ assert torch.equal(first.data["dummy"].data, item["dummy"].data)
409
+
410
+ # Request 2 (identical, fully prefix-covered): the sender hit takes
411
+ # writer references (touch + get_cached) and returns the address
412
+ # descriptor; the scheduler strips the covered uncached payload to
413
+ # data=None.
414
+ p0.touch_sender_cache_item(mm_hash)
415
+ address_item_2, _ = p0.get_and_update_item(None, mm_hash)
416
+ covered_uncached = _feature("image_B", None)
417
+ covered_cached = _feature(mm_hash, address_item_2)
418
+
419
+ # Pre-fix this asserted in touch_receiver_cache_item and terminated
420
+ # EngineCore.
421
+ p1.get_and_update_features([covered_uncached, covered_cached])
422
+
423
+ assert covered_uncached.data is None
424
+ # The address item is resolved to the cached payload.
425
+ assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data)
426
+
427
+ # The hit's writer references were acknowledged by the worker, so the
428
+ # object remains reclaimable instead of permanently protected.
429
+ p0._shm_cache.free_unused()
430
+ assert not p0._shm_cache.is_cached(mm_hash)
431
+ finally:
432
+ p1._shm_cache.close()
433
+ p0.close()
434
+
435
+
436
def _run_test_cache_eviction_lru(
437
p0_cache: BaseMultiModalProcessorCache,
438
p1_cache: BaseMultiModalReceiverCache,
tests/v1/core/test_output.py
+23
@@ -139,3 +139,26 @@ def test_strip_covered_mm_data_mrope() -> None:
139
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
140
# original list is not mutated
141
assert set(covered.data.keys()) == {"pixel_values", "image_grid_thw"}
142
+
143
+
144
+def test_strip_covered_mm_data_shm_address_item() -> None:
145
+ """SHM address descriptors must survive stripping even when covered: the
146
+ worker needs the address to resolve the payload and to acknowledge the
147
+ sender's writer reference. Stripping it crashed the SHM receiver cache on
148
+ the second identical request (vllm-project/vllm#54994)."""
149
+ address_item = MultiModalKwargsItem(
150
+ {
151
+ "address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
152
+ "monotonic_id": MultiModalFieldElem(data=7, field=MultiModalBatchedField()),
153
+ }
154
+ )
155
+ feature = MultiModalFeatureSpec(
156
+ data=address_item,
157
+ mm_position=PlaceholderRange(offset=0, length=100),
158
+ identifier="shm_item",
159
+ modality="image",
160
+ )
161
+
162
+ stripped = strip_covered_mm_data([feature], num_computed_tokens=250)
163
+
164
+ assert stripped[0].data is address_item
vllm/multimodal/cache.py
+12
@@ -794,6 +794,18 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache):
794
reader_lock=shared_worker_lock,
795
)
796
797
+ @override
798
+ def get_and_update_features(
799
+ self, mm_features: list["MultiModalFeatureSpec"]
800
+ ) -> list["MultiModalFeatureSpec"]:
801
+ # strip_covered_mm_data preserves address items, so None here represents
802
+ # a stripped uncached payload and has no SHM reference to acknowledge.
803
+ features_with_data = [
804
+ feature for feature in mm_features if feature.data is not None
805
+ ]
806
+ super().get_and_update_features(features_with_data)
807
+ return mm_features
808
+
809
@override
810
def get_and_update_item(
811
self,
vllm/multimodal/utils.py
+9
-4
@@ -221,15 +221,20 @@ def strip_covered_mm_data(
221
"""Drop the tensor data of mm items whose placeholder span is fully inside
222
a prefix-cache-covered region: no encoder run can be scheduled for them,
223
so the workers never consume the payload fields. M-RoPE and XD-RoPE models
224
- are the exception: the worker computes positions for the whole prompt from
225
- the CPU-side metadata fields (e.g. grid dims), so those are kept. The
224
+ keep CPU-side metadata fields used to compute positions. SHM address items
225
+ are also kept so workers can balance the sender's reference count. The
226
scheduler-side ``Request`` keeps the full features."""
227
if not mm_features or num_computed_tokens == 0:
228
return mm_features
229
230
def maybe_strip(f: MultiModalFeatureSpec) -> MultiModalFeatureSpec:
231
- if f.data is None or (
232
- f.mm_position.offset + f.mm_position.length > num_computed_tokens
231
+ if (
232
+ f.data is None
233
+ # SHM address items must reach the worker so it can acknowledge
234
+ # the sender's reference count before the item is evicted. This
235
+ # mirrors how the SHM receiver identifies them ("address" in item).
236
+ or "address" in f.data
237
+ or (f.mm_position.offset + f.mm_position.length > num_computed_tokens)
238
):
239
return f
240