[Bugfix][V1] Keep an encoder cache entry until its last occurrence is freed (#54284)
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
Hotragn Pettugani committed
Aug 29, 2026 at 01:08 UTC
3958a420f0ec29a2fe2e760a42418e61c1f0f5f7
2 files changed
+48
tests/v1/core/test_encoder_cache_manager.py
+37
@@ -349,6 +349,43 @@ def test_free_request_with_duplicate_mm_hashes():
349
assert "r1" not in manager.request_cached_ids
350
351
352
+def test_duplicate_mm_hash_stays_referenced_until_last_free():
353
+ """`cached` holds one reference per request, not per position, so freeing
354
+ the first of two occurrences must not release the entry."""
355
+ manager = EncoderCacheManager(cache_size=20)
356
+ req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
357
+
358
+ manager.allocate(req, 0)
359
+ assert manager.check_and_update_cache(req, 1)
360
+
361
+ manager.free_encoder_input(req, 0)
362
+ assert manager.cached["imgA"] == {"r1"}
363
+ assert "imgA" not in manager.freeable
364
+ assert manager.num_freeable_slots == 16
365
+
366
+ manager.free_encoder_input(req, 1)
367
+ assert not manager.cached["imgA"]
368
+ assert manager.freeable["imgA"] == 4
369
+ assert manager.num_freeable_slots == 20
370
+
371
+
372
+def test_duplicate_mm_hash_is_not_evicted_before_last_use():
373
+ """An item repeated within one request (an image carried across
374
+ conversation turns) must not become reclaimable capacity while a later
375
+ occurrence still has to be spliced in, or the encoder recomputes it."""
376
+ manager = EncoderCacheManager(cache_size=8)
377
+ req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
378
+ other = MockRequest("r2", ["imgB"], [8])
379
+
380
+ manager.allocate(req, 0)
381
+ assert manager.check_and_update_cache(req, 1)
382
+ manager.free_encoder_input(req, 0)
383
+
384
+ assert not manager.can_allocate(other, 0, int(1e9), 0)
385
+ assert manager.check_and_update_cache(req, 1)
386
+ assert manager.get_freed_mm_hashes() == []
387
+
388
+
389
def test_encoder_decoder_cache_manager_reset():
390
manager = EncoderDecoderCacheManager(cache_size=20)
391
vllm/v1/core/encoder_cache_manager.py
+11
@@ -242,6 +242,17 @@ class EncoderCacheManager:
242
# The mm_hash not in cache or the req_id set is empty
243
if not self.cached.get(mm_hash, None):
244
return
245
+ # `cached` counts referencing requests, not positions, so one request
246
+ # that repeats an item (an image carried across conversation turns) has
247
+ # a single reference covering every occurrence. Hold it until the last
248
+ # occurrence is freed: dropping it at the first makes the entry
249
+ # evictable while the request still needs it, and the encoder then
250
+ # recomputes an item it already has.
251
+ if any(
252
+ request.mm_features[other_id].identifier == mm_hash
253
+ for other_id in self.request_cached_ids.get(req_id, ())
254
+ ):
255
+ return
256
self.cached[mm_hash].discard(req_id)
257
if not self.cached[mm_hash]:
258
num_encoder_embeds = request.get_num_encoder_embeds(input_id)