A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 01:45 UTC
[Bugfix] Clear P0 mm sender cache on sleep/pause to fix mm_hash desync (#43001)
Signed-off-by: Tobias Wasner <wasnertobias@gmail.com>
Tobias Wasner committed
May 22, 2026 at 12:54 UTC
4658bf882b881287fc85797a23037aa91740b7a7
3 files changed
+45
tests/multimodal/test_cache.py
+39
@@ -549,3 +549,42 @@ def test_processor_cache_shared_across_loras():
549
550
receiver_cache.get_and_update_features([feature_lora_b])
551
assert feature_lora_b.data == item_data
552
+
553
+
554
+_SLEEP_VISION_PROMPT = (
555
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
556
+ "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
557
+ "What is in the image?<|im_end|>\n"
558
+ "<|im_start|>assistant\n"
559
+)
560
+
561
+
562
+@pytest.mark.skipif(
563
+ not torch.cuda.is_available(),
564
+ reason="sleep mode regression requires a CUDA GPU",
565
+)
566
+def test_sleep_wake_preserves_mm_cache_consistency():
567
+ """Regression for vllm-project/vllm#42995."""
568
+ from vllm import LLM, SamplingParams
569
+ from vllm.assets.image import ImageAsset
570
+
571
+ image = ImageAsset("stop_sign").pil_image
572
+ prompt = {
573
+ "prompt": _SLEEP_VISION_PROMPT,
574
+ "multi_modal_data": {"image": image},
575
+ }
576
+ sampling_params = SamplingParams(temperature=0, max_tokens=8)
577
+
578
+ llm = LLM(
579
+ model="Qwen/Qwen2-VL-2B-Instruct",
580
+ enable_sleep_mode=True,
581
+ enforce_eager=True,
582
+ gpu_memory_utilization=0.5,
583
+ max_model_len=2048,
584
+ )
585
+
586
+ llm.generate([prompt], sampling_params)
587
+ llm.sleep(level=1)
588
+ llm.wake_up()
589
+ output2 = llm.generate([prompt], sampling_params)
590
+ assert output2[0].outputs[0].text
vllm/v1/engine/async_llm.py
+4
@@ -781,6 +781,8 @@ class AsyncLLM(EngineClient):
781
stacklevel=2,
782
)
783
mode = "wait"
784
+ if clear_cache:
785
+ await self.renderer.clear_mm_cache_async()
786
await self.engine_core.pause_scheduler_async(mode=mode, clear_cache=clear_cache)
787
# Small sleep to help ensure that final outputs from any in-flight requests are
788
# returned prior to this method returning. These outputs come out of the engine
@@ -927,6 +929,8 @@ class AsyncLLM(EngineClient):
929
await self.engine_core.reset_encoder_cache_async()
930
931
async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
932
+ if level >= 1:
933
+ await self.renderer.clear_mm_cache_async()
934
await self.engine_core.sleep_async(level, mode)
935
936
if self.logger_manager is not None:
vllm/v1/engine/llm_engine.py
+2
@@ -350,6 +350,8 @@ class LLMEngine:
350
self.engine_core.reset_encoder_cache()
351
352
def sleep(self, level: int = 1, mode: PauseMode = "abort"):
353
+ if level >= 1:
354
+ self.renderer.clear_mm_cache()
355
self.engine_core.sleep(level, mode)
356
357
if self.logger_manager is not None: