A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 22:30 UTC
[LMCache MP Connector] Add num_lmcache_extra_cached_token in KVTransferParams (#39843)
Signed-off-by: aeon-x <talexcao@gmail.com>
Cao Qian committed
Apr 20, 2026 at 13:42 UTC
8b1f3bebcab8f501e13183f84b20635cf336ccc1
1 file changed
+20
-1
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+20
-1
@@ -930,12 +930,31 @@ class LMCacheMPConnector(KVConnectorBase_V1):
930
Optional KVTransferParams to be included in the request outputs
931
returned by the engine.
932
"""
933
+
934
+ params: dict[str, Any] | None = getattr(request, "kv_transfer_params", None)
935
+ return_params: dict[str, Any] | None = {} if params is not None else None
936
+
937
+ if (
938
+ params is not None
939
+ and return_params is not None
940
+ and "num_lmcache_extra_cached_tokens" in params
941
+ ):
942
+ request_tracker = self._get_request_tracker(request.request_id)
943
+ num_extra_cached_blocks = max(
944
+ 0,
945
+ request_tracker.num_lmcache_hit_blocks
946
+ - request_tracker.num_vllm_hit_blocks,
947
+ )
948
+ return_params["num_lmcache_extra_cached_tokens"] = (
949
+ num_extra_cached_blocks * self.vllm_block_size
950
+ )
951
+
952
# Clean up request tracker to prevent memory leak
953
self._cleanup_request_tracker(request.request_id)
954
# Notify LMCache to end the session for this request
955
self.scheduler_adapter.end_session(request.request_id)
956
938
- return True, None
957
+ return True, return_params
958
959
def take_events(self) -> Iterable["KVCacheEvent"]:
960
"""