A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 06:00 UTC
fix(lmcache): correct store for cached requests and num_scheduled_tokens in lmcache_mp_connector.py (#39655)
Signed-off-by: baoloongmao <baoloongmao@tencent.com>
maobaolong committed
Apr 13, 2026 at 11:29 UTC
2a3c32ce674950f94fdd447979e4621267125e41
1 file changed
+14
-3
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+14
-3
@@ -304,10 +304,20 @@ class LMCacheMPRequestMetadata:
304
# NOTE: the invariant here is that `num_stored_blocks` should
305
# always be a multiple of `blocks_in_chunk`
306
# TODO: This should be checked everytime we update the num_stored_blocks
307
+ #
308
+ # Why computed_blocks includes num_lmcache_hit_blocks:
309
+ #
310
+ # Include lmcache-hit blocks so that the upper bound
311
+ # matches num_stored_blocks (which already covers
312
+ # them). Hit blocks are NOT re-stored.
313
+ computed_blocks = (
314
+ tracker.num_scheduled_tokens // vllm_block_size
315
+ + tracker.num_lmcache_hit_blocks
316
+ )
317
min_available_blocks = min(
318
len(tracker.block_hashes),
319
len(tracker.allocated_block_ids),
310
- tracker.num_scheduled_tokens // vllm_block_size,
320
+ computed_blocks,
321
)
322
num_staging_blocks = min_available_blocks - tracker.num_stored_blocks
323
num_chunks = num_staging_blocks // blocks_in_chunk
@@ -1010,8 +1020,9 @@ class LMCacheMPConnector(KVConnectorBase_V1):
1020
if request_id not in cached_reqs.resumed_req_ids:
1021
request_tracker.append_block_ids(new_block_ids)
1022
1013
- # Update new scheduled tokens
1014
- num_new_tokens = cached_reqs.num_computed_tokens[idx]
1023
+ # Use the incremental num_scheduled_tokens to
1024
+ # stay consistent with _process_new_requests.
1025
+ num_new_tokens = scheduler_output.num_scheduled_tokens[request_id]
1026
request_tracker.increase_num_scheduled_tokens(num_new_tokens)
1027
1028
r_meta = LMCacheMPRequestMetadata.GetStoreMetadata(