A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[P/D][V1] Support dynamic loading of external KV connector implementations (#18142)
Signed-off-by: David Ben-David <davidb@pliops.com> Co-authored-by: David Ben-David <davidb@pliops.com>
David Ben-David committed
May 17, 2025 at 09:40 UTC
3e0d435027f7585afbbc8b44ca1e386c0a88ae47
2 files changed
+15
-2
vllm/config.py
+4
@@ -3524,6 +3524,10 @@ class KVTransferConfig:
3524
kv_connector_extra_config: dict[str, Any] = field(default_factory=dict)
3525
"""any extra config that the connector may need."""
3526
3527
+ kv_connector_module_path: Optional[str] = None
3528
+ """The Python module path to dynamically load the KV connector from.
3529
+ Only supported in V1."""
3530
+
3531
def compute_hash(self) -> str:
3532
"""
3533
WARNING: Whenever a new field is added to this config,
vllm/distributed/kv_transfer/kv_connector/factory.py
+11
-2
@@ -58,8 +58,17 @@ class KVConnectorFactory:
58
raise ValueError("Attempting to initialize a V1 Connector, "
59
f"but found {envs.VLLM_USE_V1=}")
60
61
- connector_name = config.kv_transfer_config.kv_connector
62
- connector_cls = cls._registry[connector_name]()
61
+ kv_transfer_config = config.kv_transfer_config
62
+ connector_name = kv_transfer_config.kv_connector
63
+ if connector_name in cls._registry:
64
+ connector_cls = cls._registry[connector_name]()
65
+ else:
66
+ connector_module_path = kv_transfer_config.kv_connector_module_path
67
+ if connector_module_path is None:
68
+ raise ValueError(
69
+ f"Unsupported connector type: {connector_name}")
70
+ connector_module = importlib.import_module(connector_module_path)
71
+ connector_cls = getattr(connector_module, connector_name)
72
assert issubclass(connector_cls, KVConnectorBase_V1)
73
logger.info("Creating v1 connector with name: %s", connector_name)
74
# NOTE(Kuntai): v1 connector is explicitly separated into two roles.