A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 03:30 UTC
[KVConnector][LMCache] Propagate cache_salt through MP connector for per-user cache isolation (#39837)
Signed-off-by: royyhuang <royyhuang@gmail.com> Signed-off-by: royyhuang <roy.y.huang@gmail.com>
Roy Huang committed
Apr 15, 2026 at 09:10 UTC
ed333105520c9610daa17cfe6be21383513b9c34
1 file changed
+17
-2
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+17
-2
@@ -215,8 +215,11 @@ class LMCacheMPRequestTracker:
215
# Main state
216
state: LMCacheMPRequestState = LMCacheMPRequestState.PREFETCHING
217
218
+ cache_salt: str = ""
219
+
220
def __init__(self, request: "Request"):
221
self.request_id = request.request_id
222
+ self.cache_salt: str = request.cache_salt or ""
223
self.all_token_ids = request.all_token_ids
224
self.block_hashes = ConstantList(request.block_hashes)
225
self.allocated_block_ids = []
@@ -289,6 +292,7 @@ class LMCacheMPRequestMetadata:
292
request_id: str
293
direction: Literal["STORE", "RETRIEVE"]
294
op: LoadStoreOp
295
+ cache_salt: str = ""
296
297
@staticmethod
298
def GetStoreMetadata(
@@ -355,6 +359,7 @@ class LMCacheMPRequestMetadata:
359
request_id=tracker.request_id,
360
direction="STORE",
361
op=op,
362
+ cache_salt=tracker.cache_salt,
363
)
364
365
# Update the request tracker
@@ -421,6 +426,7 @@ class LMCacheMPRequestMetadata:
426
request_id=tracker.request_id,
427
direction="RETRIEVE",
428
op=op,
429
+ cache_salt=tracker.cache_salt,
430
)
431
return ret
432
@@ -569,12 +575,14 @@ class LMCacheMPConnector(KVConnectorBase_V1):
575
576
request_ids = []
577
ops = []
578
+ cache_salts = []
579
580
for meta in metadata.requests:
581
if meta.direction != "RETRIEVE":
582
continue
583
request_ids.append(meta.request_id)
584
ops.append(meta.op)
585
+ cache_salts.append(meta.cache_salt)
586
587
if len(request_ids) == 0:
588
return
@@ -583,7 +591,9 @@ class LMCacheMPConnector(KVConnectorBase_V1):
591
event = torch.cuda.Event(interprocess=True)
592
event.record()
593
586
- self.worker_adapter.batched_submit_retrieve_requests(request_ids, ops, event)
594
+ self.worker_adapter.batched_submit_retrieve_requests(
595
+ request_ids, ops, event, cache_salts=cache_salts
596
+ )
597
598
def wait_for_layer_load(self, layer_name: str) -> None:
599
"""
@@ -640,11 +650,13 @@ class LMCacheMPConnector(KVConnectorBase_V1):
650
651
request_ids = []
652
ops = []
653
+ cache_salts = []
654
for meta in metadata.requests:
655
if meta.direction != "STORE":
656
continue
657
request_ids.append(meta.request_id)
658
ops.append(meta.op)
659
+ cache_salts.append(meta.cache_salt)
660
661
if len(request_ids) == 0:
662
return
@@ -653,7 +665,9 @@ class LMCacheMPConnector(KVConnectorBase_V1):
665
event = torch.cuda.Event(interprocess=True)
666
event.record()
667
656
- self.worker_adapter.batched_submit_store_requests(request_ids, ops, event)
668
+ self.worker_adapter.batched_submit_store_requests(
669
+ request_ids, ops, event, cache_salts=cache_salts
670
+ )
671
672
def get_finished(
673
self, finished_req_ids: set[str]
@@ -755,6 +769,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):
769
self.scheduler_adapter.maybe_submit_lookup_request(
770
request.request_id,
771
token_ids=list(request.all_token_ids),
772
+ cache_salt=tracker.cache_salt,
773
)
774
775
ret = self.scheduler_adapter.check_lookup_result(request.request_id)