A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 02:45 UTC
[Bugfix][LMCache][KVConnector] fix potential memory leak in LMCache multiprocess mode (#35931)
Roy Huang committed
Mar 7, 2026 at 13:52 UTC
63298ee17350e4eda3f574eab16286bc405b23a6
1 file changed
+28
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+28
@@ -721,6 +721,34 @@ class LMCacheMPConnector(KVConnectorBase_V1):
721
# Clean up lookup future in scheduler adapter
722
self.scheduler_adapter.cleanup_lookup_result(request.request_id)
723
724
+ # Free locks on chunks that vLLM already computed and won't
725
+ # retrieve from LMCache.
726
+ if tracker.num_lmcache_hit_blocks > 0:
727
+ if not condition:
728
+ # No retrieve needed — free ALL locked chunks
729
+ free_end = tracker.num_lmcache_hit_blocks * self.vllm_block_size
730
+ else:
731
+ # Note(Roy): Boundary misalignment between vLLM blocks and LMCache
732
+ # blocks is handled in free_lookup_locks. It makes sure that if
733
+ # the last vLLM computed block ends in the middle of a LMCache
734
+ # block, the end LMCache block is not freed (i.e., floor division)
735
+ # since it will still be needed by vLLM and such block's lock will
736
+ # be freed by vLLM's retrieve.
737
+ free_end = tracker.num_vllm_hit_blocks * self.vllm_block_size
738
+
739
+ if free_end > 0:
740
+ self.scheduler_adapter.free_lookup_locks(
741
+ token_ids=list(tracker.all_token_ids),
742
+ start=0,
743
+ end=free_end,
744
+ request_id=request.request_id,
745
+ )
746
+ logger.debug(
747
+ "Free locks of tokens %d-%d since it is cached by vLLM.",
748
+ 0,
749
+ free_end,
750
+ )
751
+
752
def build_connector_meta(
753
self, scheduler_output: SchedulerOutput
754
) -> KVConnectorMetadata: