A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[LMCache] Fault Tolerance Mechanism (#36586)
Signed-off-by: Oasis-Git <ayw.sirius19@gmail.com>
Yuwei An committed
Mar 11, 2026 at 20:54 UTC
2ef69456f5a0078bd8a8614b1cb6376e730e4d20
1 file changed
+38
-6
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+38
-6
@@ -101,7 +101,11 @@ def extract_world_size_and_kv_rank(
101
102
103
def create_scheduler_adapter(
104
- server_url: str, zmq_context: zmq.Context, vllm_config: VllmConfig
104
+ server_url: str,
105
+ zmq_context: zmq.Context,
106
+ vllm_config: VllmConfig,
107
+ mq_timeout: float,
108
+ heartbeat_interval: float,
109
) -> LMCacheMPSchedulerAdapter:
110
world_size, kv_rank = extract_world_size_and_kv_rank(
111
vllm_config.parallel_config.world_size,
@@ -123,12 +127,18 @@ def create_scheduler_adapter(
127
world_size,
128
kv_rank,
129
vllm_config.cache_config.block_size,
130
+ mq_timeout=mq_timeout,
131
+ heartbeat_interval=heartbeat_interval,
132
**kwargs,
133
)
134
135
136
def create_worker_adapter(
131
- server_url: str, zmq_context: zmq.Context, vllm_config: VllmConfig
137
+ server_url: str,
138
+ zmq_context: zmq.Context,
139
+ vllm_config: VllmConfig,
140
+ mq_timeout: float,
141
+ heartbeat_interval: float,
142
) -> LMCacheMPWorkerAdapter:
143
world_size, kv_rank = extract_world_size_and_kv_rank(
144
vllm_config.parallel_config.world_size,
@@ -142,6 +152,8 @@ def create_worker_adapter(
152
world_size,
153
kv_rank,
154
vllm_config.cache_config.block_size,
155
+ mq_timeout=mq_timeout,
156
+ heartbeat_interval=heartbeat_interval,
157
)
158
159
@@ -413,6 +425,9 @@ class LMCacheMPConnector(KVConnectorBase_V1):
425
Extra configs (kv_transfer_config.extra_config):
426
- lmcache.mp.host: the host of the LMCache server.
427
- lmcache.mp.port: the port of the LMCache server.
428
+ - lmcache.mp.mq_timeout: timeout (seconds) for message queue requests.
429
+ - lmcache.mp.heartbeat_interval: interval (seconds) between server
430
+ heartbeat pings.
431
"""
432
433
def __init__(
@@ -430,17 +445,35 @@ class LMCacheMPConnector(KVConnectorBase_V1):
445
server_port = vllm_config.kv_transfer_config.get_from_extra_config(
446
"lmcache.mp.port", 5555
447
)
448
+ mq_timeout = float(
449
+ vllm_config.kv_transfer_config.get_from_extra_config(
450
+ "lmcache.mp.mq_timeout", 300.0
451
+ )
452
+ )
453
+ heartbeat_interval = float(
454
+ vllm_config.kv_transfer_config.get_from_extra_config(
455
+ "lmcache.mp.heartbeat_interval", 10.0
456
+ )
457
+ )
458
459
server_url = f"{server_host}:{server_port}"
460
zmq_context = zmq.Context.instance()
461
if self.role == KVConnectorRole.SCHEDULER:
462
self.scheduler_adapter = create_scheduler_adapter(
438
- server_url, zmq_context, vllm_config
463
+ server_url,
464
+ zmq_context,
465
+ vllm_config,
466
+ mq_timeout,
467
+ heartbeat_interval,
468
)
469
self.request_trackers: dict[str, LMCacheMPRequestTracker] = {}
470
elif self.role == KVConnectorRole.WORKER:
471
self.worker_adapter = create_worker_adapter(
443
- server_url, zmq_context, vllm_config
472
+ server_url,
473
+ zmq_context,
474
+ vllm_config,
475
+ mq_timeout,
476
+ heartbeat_interval,
477
)
478
else:
479
raise ValueError(f"Unknown KVConnectorRole: {self.role}")
@@ -616,8 +649,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):
649
- Sync loading: failed blocks should be reported in the forward
650
pass in which they are detected.
651
"""
619
- # TODO: add error tracking
620
- return set()
652
+ return self.worker_adapter.get_block_ids_with_load_errors()
653
654
def shutdown(self):
655
"""