A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 14:04 UTC
[LMCacheMPConnector] Prioritize importing the lmcache_mp_connector from lmcache (#42596)
Signed-off-by: idellzheng <idellzheng@tencent.com>
chunxiaozheng committed
May 16, 2026 at 01:46 UTC
f45c21088509344e3201242a8b2e69e2b267cfb3
1 file changed
+37
-1
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py
+37
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import enum
4
+import os
5
from collections.abc import Iterable
6
from dataclasses import dataclass, field
7
from typing import TYPE_CHECKING, Any, Literal
@@ -460,7 +461,7 @@ class LMCacheMPConnectorMetadata(KVConnectorMetadata):
461
return self.__str__()
462
463
463
-class LMCacheMPConnector(KVConnectorBase_V1):
464
+class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
465
"""
466
The connector for LMCache multi-process mode.
467
@@ -1188,3 +1189,38 @@ class LMCacheMPConnector(KVConnectorBase_V1):
1189
"[KVConnector] Cleaned up request_tracker for request %s",
1190
request_id,
1191
)
1192
+
1193
+
1194
+# At module load time, prefer the external LMCacheMPConnector shipped with the
1195
+# ``lmcache`` package. This avoids forcing users to set
1196
+# ``kv_connector_module_path`` when they only configure ``kv_connector``. If
1197
+# the external module is unavailable (e.g. older lmcache version that does
1198
+# not ship this submodule, or any import error), fall back to the builtin
1199
+# implementation defined above.
1200
+def _resolve_lmcache_mp_connector() -> type[KVConnectorBase_V1]:
1201
+ if os.environ.get("LMCACHE_USE_UPSTREAM_MP"):
1202
+ logger.info(
1203
+ "Force use builtin LMCacheMPConnectorUpstream in vLLM.",
1204
+ )
1205
+ return LMCacheMPConnectorUpstream
1206
+
1207
+ try:
1208
+ from lmcache.integration.vllm.lmcache_mp_connector import (
1209
+ LMCacheMPConnector as _ExternalLMCacheMPConnector,
1210
+ )
1211
+
1212
+ logger.info(
1213
+ "Using external LMCacheMPConnector from "
1214
+ "lmcache.integration.vllm.lmcache_mp_connector"
1215
+ )
1216
+ return _ExternalLMCacheMPConnector
1217
+ except ImportError as e:
1218
+ logger.info(
1219
+ "External LMCacheMPConnector is not available (%s), "
1220
+ "falling back to builtin implementation in vLLM.",
1221
+ e,
1222
+ )
1223
+ return LMCacheMPConnectorUpstream
1224
+
1225
+
1226
+LMCacheMPConnector = _resolve_lmcache_mp_connector()