A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc] KV cache transfer connector registry (#11481)
Signed-off-by: KuntaiDu <kuntai@uchicago.edu>
Kuntai Du committed
Dec 29, 2024 at 10:08 UTC
faef77c0d69c5429182f475a57127676e6bcb230
2 files changed
+38
-18
vllm/config.py
-8
@@ -2559,14 +2559,6 @@ class KVTransferConfig(BaseModel):
2559
return KVTransferConfig.model_validate_json(cli_value)
2560
2561
def model_post_init(self, __context: Any) -> None:
2562
- supported_kv_connector = ["PyNcclConnector", "MooncakeConnector"]
2563
- if all([
2564
- self.kv_connector is not None, self.kv_connector
2565
- not in supported_kv_connector
2566
- ]):
2567
- raise ValueError(f"Unsupported kv_connector: {self.kv_connector}. "
2568
- f"Supported connectors are "
2569
- f"{supported_kv_connector}.")
2562
2563
if self.kv_role is not None and self.kv_role not in [
2564
"kv_producer", "kv_consumer", "kv_both"
vllm/distributed/kv_transfer/kv_connector/factory.py
+38
-10
@@ -1,4 +1,5 @@
1
-from typing import TYPE_CHECKING
1
+import importlib
2
+from typing import TYPE_CHECKING, Callable, Dict, Type
3
4
from .base import KVConnectorBase
5
@@ -7,14 +8,41 @@ if TYPE_CHECKING:
8
9
10
class KVConnectorFactory:
11
+ _registry: Dict[str, Callable[[], Type[KVConnectorBase]]] = {}
12
11
- @staticmethod
12
- def create_connector(rank: int, local_rank: int,
13
+ @classmethod
14
+ def register_connector(cls, name: str, module_path: str,
15
+ class_name: str) -> None:
16
+ """Register a connector with a lazy-loading module and class name."""
17
+ if name in cls._registry:
18
+ raise ValueError(f"Connector '{name}' is already registered.")
19
+
20
+ def loader() -> Type[KVConnectorBase]:
21
+ module = importlib.import_module(module_path)
22
+ return getattr(module, class_name)
23
+
24
+ cls._registry[name] = loader
25
+
26
+ @classmethod
27
+ def create_connector(cls, rank: int, local_rank: int,
28
config: "VllmConfig") -> KVConnectorBase:
14
- supported_kv_connector = ["PyNcclConnector", "MooncakeConnector"]
15
- if config.kv_transfer_config.kv_connector in supported_kv_connector:
16
- from .simple_connector import SimpleConnector
17
- return SimpleConnector(rank, local_rank, config)
18
- else:
19
- raise ValueError(f"Unsupported connector type: "
20
- f"{config.kv_connector}")
29
+ connector_name = config.kv_transfer_config.kv_connector
30
+ if connector_name not in cls._registry:
31
+ raise ValueError(f"Unsupported connector type: {connector_name}")
32
+
33
+ connector_cls = cls._registry[connector_name]()
34
+ return connector_cls(rank, local_rank, config)
35
+
36
+
37
+# Register various connectors here.
38
+# The registration should not be done in each individual file, as we want to
39
+# only load the files corresponding to the current connector.
40
+KVConnectorFactory.register_connector(
41
+ "PyNcclConnector",
42
+ "vllm.distributed.kv_transfer.kv_connector.simple_connector",
43
+ "SimpleConnector")
44
+
45
+KVConnectorFactory.register_connector(
46
+ "MooncakeConnector",
47
+ "vllm.distributed.kv_transfer.kv_connector.simple_connector",
48
+ "SimpleConnector")