A high-throughput and memory-efficient inference and serving engine for LLMs
[KV Connector][Mooncake] Add cache_prefix to namespace store keys (#45767)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Dao007forever committed
Jun 16, 2026 at 10:24 UTC
44b25127670b16bd5c1e06a093ddd0f137c868dc
4 files changed
+43
docs/features/mooncake_store_connector_usage.md
+1
@@ -205,6 +205,7 @@ the vLLM JSON config.
205
- `load_async` (bool): Enable asynchronous loading for better compute-I/O overlap. Default: `true`.
206
- `enable_cross_layers_blocks` (bool): Enable cross-layer block packing for reduced store operations. Default: `false`.
207
- `lookup_rpc_port` (int): Custom port for the ZMQ lookup RPC socket. Default: `0`.
208
+- `cache_prefix` (str): Namespace prepended to every store key. Lets separate deployments share one Mooncake master without polluting each other — instances configured with different prefixes never see each other's cached blocks, even for identical prompts. All instances that should share a prefix cache must use the same value. Default: `""` (no prefix; keys are byte-identical to the unprefixed format).
209
210
## Notes
211
tests/v1/kv_connector/unit/test_mooncake_store_worker.py
+26
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
26
ChunkedTokenDatabase,
27
KeyMetadata,
28
LoadSpec,
29
+ PoolKey,
30
ReqMeta,
31
)
32
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
@@ -240,6 +241,31 @@ def _patch_worker_runtime(monkeypatch, *, local_ip: str = "10.0.0.7") -> None:
241
monkeypatch.setattr(worker, "get_ip", lambda: local_ip)
242
243
244
+def test_pool_key_to_string_without_prefix_is_unchanged():
245
+ """Default (empty) cache_prefix keeps keys byte-identical to the
246
+ historical unprefixed format so existing deployments keep their hits."""
247
+ key = PoolKey(KeyMetadata("test-model", 0, 0, 0, 0), "deadbeef")
248
+ assert (
249
+ key.to_string() == "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef"
250
+ )
251
+
252
+
253
+def test_pool_key_cache_prefix_namespaces_and_disambiguates():
254
+ """A non-empty cache_prefix is prepended, and two instances with
255
+ different prefixes never collide on identical block hashes."""
256
+ md_a = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depA")
257
+ md_b = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depB")
258
+
259
+ key_a = PoolKey(md_a, "deadbeef")
260
+ key_b = PoolKey(md_b, "deadbeef")
261
+
262
+ assert key_a.to_string() == (
263
+ "depA@test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef"
264
+ )
265
+ assert key_a.to_string() != key_b.to_string()
266
+ assert hash(key_a) != hash(key_b)
267
+
268
+
269
def test_default_local_buffer_size_matches_pr40900():
270
"""PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode
271
patch preserves it (and the JSON key) so unchanged PR-40900 configs work."""
vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py
+11
@@ -33,6 +33,10 @@ class KeyMetadata:
33
dcp_rank: int
34
pp_rank: int
35
group_id: int = 0
36
+ # Optional namespace prepended to every key. Lets separate deployments
37
+ # share one Mooncake master without colliding on identical block hashes.
38
+ # Empty (the default) keeps keys byte-identical to the unprefixed format.
39
+ cache_prefix: str = ""
40
41
42
@dataclass(order=True)
@@ -45,6 +49,7 @@ class PoolKey:
49
def __hash__(self):
50
return hash(
51
(
52
+ self.key_metadata.cache_prefix,
53
self.key_metadata.model_name,
54
self.key_metadata.tp_rank,
55
self.key_metadata.pcp_rank,
@@ -56,7 +61,13 @@ class PoolKey:
61
)
62
63
def to_string(self) -> str:
64
+ prefix = (
65
+ f"{self.key_metadata.cache_prefix}@"
66
+ if self.key_metadata.cache_prefix
67
+ else ""
68
+ )
69
return (
70
+ f"{prefix}"
71
f"{self.key_metadata.model_name}"
72
f"@tp_rank:{self.key_metadata.tp_rank}"
73
f"@pcp{self.key_metadata.pcp_rank}"
vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py
+5
@@ -982,6 +982,11 @@ class MooncakeStoreWorker:
982
pcp_rank=self.pcp_rank,
983
dcp_rank=self.dcp_rank,
984
pp_rank=self.pp_rank,
985
+ cache_prefix=str(
986
+ vllm_config.kv_transfer_config.kv_connector_extra_config.get(
987
+ "cache_prefix", ""
988
+ )
989
+ ),
990
)
991
992
# Initialize MooncakeDistributedStore with its own TransferEngine