A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 04:45 UTC
fix(lmcache): correct store for cached requests while enable prefix cache (#39719)
Signed-off-by: baoloongmao <baoloongmao@tencent.com>
maobaolong committed
Apr 15, 2026 at 04:51 UTC
b2f749dc97d59e3eb808499e88a29663f2143aa1
1 file changed
+18
-7
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+18
-7
@@ -309,14 +309,25 @@ class LMCacheMPRequestMetadata:
309
# always be a multiple of `blocks_in_chunk`
310
# TODO: This should be checked everytime we update the num_stored_blocks
311
#
312
- # Why computed_blocks includes num_lmcache_hit_blocks:
312
+ # Why computed_blocks uses max(num_vllm_hit_blocks, num_lmcache_hit_blocks):
313
#
314
- # Include lmcache-hit blocks so that the upper bound
315
- # matches num_stored_blocks (which already covers
316
- # them). Hit blocks are NOT re-stored.
317
- computed_blocks = (
318
- tracker.num_scheduled_tokens // vllm_block_size
319
- + tracker.num_lmcache_hit_blocks
314
+ # Both values represent a prefix of blocks whose KV data is already
315
+ # available (either from vLLM APC or from LMCache), so they must NOT
316
+ # be summed (that would double-count the overlapping prefix).
317
+ #
318
+ # * num_lmcache_hit_blocks: LMCache-hit blocks are already counted in
319
+ # num_stored_blocks (set during lookup), so they must be included
320
+ # here to keep the upper bound consistent. They are NOT re-stored.
321
+ # * num_vllm_hit_blocks: LMCache stores in units of chunks (N blocks),
322
+ # so num_lmcache_hit_blocks is rounded DOWN to the nearest chunk
323
+ # boundary. When vLLM APC hits more blocks than that rounded value
324
+ # (e.g. APC=44 blocks, LMCache=32 blocks after chunk alignment),
325
+ # using only num_lmcache_hit_blocks would set the upper bound too
326
+ # low and silently skip the APC-hit blocks that fall between the
327
+ # two values, causing under-storing. Taking the max ensures we
328
+ # always use the tighter (larger) of the two hit counts.
329
+ computed_blocks = tracker.num_scheduled_tokens // vllm_block_size + max(
330
+ tracker.num_vllm_hit_blocks, tracker.num_lmcache_hit_blocks
331
)
332
min_available_blocks = min(
333
len(tracker.block_hashes),