A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 19:30 UTC
[Bugfix] Fix Whisper/encoder-decoder GPU memory leak (#32789)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Jan 22, 2026 at 11:50 UTC
ea6102b85da808b23055912391977f43fbe3f227
2 files changed
+54
-5
tests/models/multimodal/generation/test_whisper.py
+43
@@ -176,3 +176,46 @@ def test_models_distributed(
176
distributed_executor_backend=distributed_executor_backend,
177
enforce_eager=False,
178
)
179
+
180
+
181
+@pytest.mark.core_model
182
+@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
183
+def test_encoder_cache_cleanup(
184
+ vllm_runner,
185
+ model: str,
186
+ input_audios,
187
+ monkeypatch,
188
+) -> None:
189
+ """Test that encoder cache is properly cleaned up after requests complete.
190
+
191
+ This is a regression test for a bug where encoder cache entries were freed
192
+ in the same scheduling step they were allocated, before the model could use
193
+ them.
194
+ """
195
+ # Set single-process mode to access the model runner's encoder cache directly
196
+ monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
197
+ check_model_available(model)
198
+
199
+ with vllm_runner(
200
+ model,
201
+ dtype="half",
202
+ max_model_len=448,
203
+ tensor_parallel_size=1,
204
+ limit_mm_per_prompt={"audio": 2},
205
+ enforce_eager=True,
206
+ ) as vllm_model:
207
+ engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
208
+ model_runner = engine_core.model_executor.driver_worker.worker.model_runner
209
+ encoder_cache = model_runner.encoder_cache
210
+
211
+ # Run multiple sequential requests to ensure cache is properly managed
212
+ for vllm_prompts, _, audios in input_audios:
213
+ vllm_model.generate_greedy(vllm_prompts, max_tokens=50, audios=audios)
214
+
215
+ # After all requests complete, encoder cache should be empty
216
+ cache_size = len(encoder_cache)
217
+ assert cache_size == 0, (
218
+ f"Encoder cache should be empty after all requests complete, "
219
+ f"but has {cache_size} entries. This indicates encoder cache "
220
+ f"entries are not being properly freed."
221
+ )
vllm/v1/core/encoder_cache_manager.py
+11
-5
@@ -357,7 +357,8 @@ class EncoderDecoderCacheManager(EncoderCacheManager):
357
def __init__(self, cache_size: int):
358
self.cache_size = cache_size
359
self.num_free_slots = cache_size
360
- self.freed: list[str] = []
360
+ self.allocated: list[str] = []
361
+ self.to_free: list[str] = []
362
363
def check_and_update_cache(self, request: Request, input_id: int) -> bool:
364
return False
@@ -383,7 +384,7 @@ class EncoderDecoderCacheManager(EncoderCacheManager):
384
self.num_free_slots -= num_encoder_embeds
385
386
mm_hash = request.mm_features[input_id].identifier
386
- self.freed.append(mm_hash)
387
+ self.allocated.append(mm_hash)
388
389
def free(self, request: Request) -> None:
390
for input_id in range(len(request.mm_features)):
@@ -393,9 +394,14 @@ class EncoderDecoderCacheManager(EncoderCacheManager):
394
return set(range(len(request.mm_features)))
395
396
def get_freed_mm_hashes(self) -> list[str]:
396
- freed = self.freed
397
- self.freed = []
398
- return freed
397
+ # As encoder cache is not used for enc-dec models, we can free the entries here
398
+ # The actual free happens in the runner, *before* the model is executed.
399
+ # Therefore, `freeable` acts as a buffer to free the entries only after the
400
+ # model is executed, mimicking the state transition of `EncoderCacheManager`.
401
+ to_free = self.to_free
402
+ self.to_free = self.allocated
403
+ self.allocated = []
404
+ return to_free
405
406
def free_encoder_input(self, request: Request, input_id: int) -> None:
407
num_encoder_embeds = request.get_num_encoder_embeds(input_id)