@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d263bd9df7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Core] Support disaggregated prefill with Mooncake Transfer Engine (#10884)

Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>

shangmingc committed Dec 16, 2024 at 05:28 UTC d263bd9df7b2f5586910e5d006a11ff11ba7c310
4 files changed +352 -31
vllm/config.py
+4 -3
@@ -2171,13 +2171,14 @@ class KVTransferConfig(BaseModel):
2171 return KVTransferConfig.model_validate_json(cli_value)
2172
2173 def model_post_init(self, __context: Any) -> None:
2174 + supported_kv_connector = ["PyNcclConnector", "MooncakeConnector"]
2175 if all([
2175 - self.kv_connector is not None,
2176 - self.kv_connector != "PyNcclConnector"
2176 + self.kv_connector is not None, self.kv_connector
2177 + not in supported_kv_connector
2178 ]):
2179 raise ValueError(f"Unsupported kv_connector: {self.kv_connector}. "
2180 f"Supported connectors are "
2180 - f"`PyNcclConnector`.")
2181 + f"{supported_kv_connector}.")
2182
2183 if self.kv_role is not None and self.kv_role not in [
2184 "kv_producer", "kv_consumer", "kv_both"
vllm/distributed/kv_transfer/kv_connector/factory.py
+2 -1
@@ -11,7 +11,8 @@ class KVConnectorFactory:
11 @staticmethod
12 def create_connector(rank: int, local_rank: int,
13 config: "VllmConfig") -> KVConnectorBase:
14 - if config.kv_transfer_config.kv_connector == 'PyNcclConnector':
14 + supported_kv_connector = ["PyNcclConnector", "MooncakeConnector"]
15 + if config.kv_transfer_config.kv_connector in supported_kv_connector:
16 from .simple_connector import SimpleConnector
17 return SimpleConnector(rank, local_rank, config)
18 else:
vllm/distributed/kv_transfer/kv_connector/simple_connector.py
+74 -27
@@ -2,7 +2,8 @@
2 Simple KV Cache Connector for Distributed Machine Learning Inference
3
4 The SimpleConnector transfers KV caches between prefill vLLM worker (KV cache
5 -producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe.
5 +producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe or
6 +MooncakePipe.
7
8 But the logic can be extended to support other pipe and lookup buffer.
9 """
@@ -15,7 +16,6 @@ from vllm.config import VllmConfig
16 from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase
17 from vllm.distributed.kv_transfer.kv_lookup_buffer.simple_buffer import (
18 SimpleBuffer)
18 -from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
19 from vllm.logger import init_logger
20 from vllm.sequence import IntermediateTensors
21
@@ -36,7 +36,27 @@ class SimpleConnector(KVConnectorBase):
36
37 self.config = config.kv_transfer_config
38
39 - logger.info("Initializing PyNcclConfig under kv_transfer_config %s",
39 + if self.config.kv_connector == "PyNcclConnector":
40 + from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import (
41 + PyNcclPipe)
42 + logger.info(
43 + "Initializing PyNcclConfig under kv_transfer_config %s",
44 + self.config)
45 + elif self.config.kv_connector == "MooncakeConnector":
46 + # Check if MOONCAKE_CONFIG_PATH is set
47 + import os
48 + use_mooncake_distributed_pipe = os.getenv(
49 + 'MOONCAKE_CONFIG_PATH') is not None
50 +
51 + if not use_mooncake_distributed_pipe:
52 + raise ValueError(
53 + "To use MooncakeConnector, you need to pass the ENV: "
54 + "'MOONCAKE_CONFIG_PATH=/path/to/mooncake_config.json'.")
55 + else:
56 + from vllm.distributed.kv_transfer.kv_pipe.mooncake_pipe import ( # noqa: E501
57 + MooncakePipe)
58 + logger.info(
59 + "Initializing MooncakeConfig under kv_transfer_config %s",
60 self.config)
61
62 self.lookup_buffer_size = self.config.kv_buffer_size
@@ -44,6 +64,11 @@ class SimpleConnector(KVConnectorBase):
64 self.producer_buffer: Optional[SimpleBuffer] = None
65 self.consumer_buffer: Optional[SimpleBuffer] = None
66
67 + self.producer_data_pipe: Union[PyNcclPipe, MooncakePipe]
68 + self.consumer_data_pipe: Union[PyNcclPipe, MooncakePipe]
69 + self.producer_signal_pipe: Union[PyNcclPipe, MooncakePipe]
70 + self.consumer_signal_pipe: Union[PyNcclPipe, MooncakePipe]
71 +
72 # 2 pipes for every rank in the world
73 port_offset_base = 2 * rank
74
@@ -51,17 +76,26 @@ class SimpleConnector(KVConnectorBase):
76 # and the decode vLLM only uses recv pipe
77 if self.config.is_kv_producer:
78
54 - self.producer_data_pipe = PyNcclPipe(
55 - local_rank=local_rank,
56 - config=self.config,
57 - port_offset=port_offset_base,
58 - )
59 - self.producer_signal_pipe = PyNcclPipe(
60 - local_rank=local_rank,
61 - config=self.config,
62 - port_offset=port_offset_base + 1,
63 - device="cpu",
64 - )
79 + if self.config.kv_connector == "PyNcclConnector":
80 + self.producer_data_pipe = PyNcclPipe(
81 + local_rank=local_rank,
82 + config=self.config,
83 + port_offset=port_offset_base,
84 + )
85 + self.producer_signal_pipe = PyNcclPipe(
86 + local_rank=local_rank,
87 + config=self.config,
88 + port_offset=port_offset_base + 1,
89 + device="cpu",
90 + )
91 + elif self.config.kv_connector == "MooncakeConnector":
92 + self.producer_data_pipe = MooncakePipe(
93 + local_rank=local_rank,
94 + config=self.config,
95 + )
96 + # We only need to initialize MooncakePipe once
97 + self.producer_signal_pipe = self.producer_data_pipe
98 +
99 self.producer_buffer = SimpleBuffer(self.producer_signal_pipe,
100 self.producer_data_pipe,
101 self.config.kv_buffer_size)
@@ -70,17 +104,25 @@ class SimpleConnector(KVConnectorBase):
104
105 # the current vLLM instance is KV consumer, so it needs to connect
106 # its recv pipe to the send pipe of KV producder
73 - self.consumer_data_pipe = PyNcclPipe(
74 - local_rank=local_rank,
75 - config=self.config,
76 - port_offset=port_offset_base,
77 - )
78 - self.consumer_signal_pipe = PyNcclPipe(
79 - local_rank=local_rank,
80 - config=self.config,
81 - port_offset=port_offset_base + 1,
82 - device="cpu",
83 - )
107 + if self.config.kv_connector == "PyNcclConnector":
108 + self.consumer_data_pipe = PyNcclPipe(
109 + local_rank=local_rank,
110 + config=self.config,
111 + port_offset=port_offset_base,
112 + )
113 + self.consumer_signal_pipe = PyNcclPipe(
114 + local_rank=local_rank,
115 + config=self.config,
116 + port_offset=port_offset_base + 1,
117 + device="cpu",
118 + )
119 + elif self.config.kv_connector == "MooncakeConnector":
120 + self.consumer_data_pipe = MooncakePipe(
121 + local_rank=local_rank,
122 + config=self.config,
123 + )
124 + self.consumer_signal_pipe = self.consumer_data_pipe
125 +
126 self.consumer_buffer = SimpleBuffer(
127 self.consumer_signal_pipe,
128 self.consumer_data_pipe,
@@ -260,6 +302,11 @@ class SimpleConnector(KVConnectorBase):
302
303 def close(self):
304 self.producer_data_pipe.close()
263 - self.producer_signal_pipe.close()
305 self.consumer_data_pipe.close()
265 - self.consumer_signal_pipe.close()
306 + if self.config.kv_connector == "PyNcclConnector":
307 + self.producer_signal_pipe.close()
308 + self.consumer_signal_pipe.close()
309 + elif self.config.kv_connector == "MooncakeConnector":
310 + # MooncakePipe reuses data_pipe for signal_pipe, so we only have to
311 + # close the data_pipe.
312 + pass
vllm/distributed/kv_transfer/kv_pipe/mooncake_pipe.py new
+272
@@ -0,0 +1,272 @@
1 +import json
2 +import os
3 +import pickle
4 +from concurrent.futures import ThreadPoolExecutor
5 +from dataclasses import dataclass
6 +from typing import Optional, Union
7 +
8 +import torch
9 +import zmq
10 +
11 +from vllm.config import KVTransferConfig
12 +from vllm.distributed.kv_transfer.kv_pipe.base import KVPipeBase
13 +from vllm.logger import init_logger
14 +
15 +logger = init_logger(__name__)
16 +NONE_INT = -150886311
17 +
18 +
19 +@dataclass
20 +class MooncakeTransferEngineConfig:
21 + prefill_url: str
22 + decode_url: str
23 + metadata_backend: Union[str, None]
24 + metadata_server: str
25 + protocol: str
26 + device_name: str
27 +
28 + @staticmethod
29 + def from_file(file_path: str) -> 'MooncakeTransferEngineConfig':
30 + """Load the config from a JSON file."""
31 + with open(file_path) as fin:
32 + config = json.load(fin)
33 + return MooncakeTransferEngineConfig(
34 + prefill_url=config.get("prefill_url"),
35 + decode_url=config.get("decode_url"),
36 + metadata_backend=config.get("metadata_backend", None),
37 + metadata_server=config.get("metadata_server"),
38 + protocol=config.get("protocol", "tcp"),
39 + device_name=config.get("device_name", ""),
40 + )
41 +
42 + @staticmethod
43 + def load_from_env() -> 'MooncakeTransferEngineConfig':
44 + """Load config from a file specified in the environment variable."""
45 + config_file_path = os.getenv('MOONCAKE_CONFIG_PATH')
46 + if config_file_path is None:
47 + raise ValueError(
48 + "The environment variable 'MOONCAKE_CONFIG_PATH' is not set.")
49 + return MooncakeTransferEngineConfig.from_file(config_file_path)
50 +
51 +
52 +class MooncakeTransferEngine:
53 + """Handles the transfer of data using mooncake_vllm_adaptor and ZeroMQ."""
54 +
55 + def __init__(self, kv_rank: int, local_rank: int):
56 + try:
57 + import mooncake_vllm_adaptor as mva
58 + except ImportError as e:
59 + raise ImportError(
60 + "Please install mooncake by following the instructions at "
61 + "https://github.com/kvcache-ai/Mooncake/blob/main/doc/en/build.md " # noqa: E501
62 + "to run vLLM with MooncakeConnector.") from e
63 +
64 + self.engine = mva.mooncake_vllm_adaptor()
65 + self.local_rank = local_rank
66 +
67 + try:
68 + self.config = MooncakeTransferEngineConfig.load_from_env()
69 + logger.info("Mooncake Configuration loaded successfully.")
70 + except ValueError as e:
71 + logger.error(e)
72 + raise
73 + except Exception as exc:
74 + logger.error(
75 + "An error occurred while loading the configuration: %s", exc)
76 + raise
77 + prefill_host, base_prefill_port = self.config.prefill_url.split(':')
78 + decode_host, base_decode_port = self.config.decode_url.split(':')
79 +
80 + # Avoid ports conflict when running prefill and decode on the same node
81 + if prefill_host == decode_host and \
82 + base_prefill_port == base_decode_port:
83 + base_decode_port = str(int(base_decode_port) + 100)
84 +
85 + prefill_port = int(base_prefill_port) + self.local_rank
86 + decode_port = int(base_decode_port) + self.local_rank
87 + self.prefill_url = ':'.join([prefill_host, str(prefill_port)])
88 + self.decode_url = ':'.join([decode_host, str(decode_port)])
89 +
90 + self.initialize(self.prefill_url if kv_rank == 0 else self.decode_url,
91 + self.config.metadata_server, self.config.protocol,
92 + self.config.device_name, self.config.metadata_backend)
93 +
94 + self.remote_url = (self.decode_url
95 + if kv_rank == 0 else self.prefill_url)
96 +
97 + # Initialize ZeroMQ context and sockets
98 + self.context = zmq.Context() # type: ignore[attr-defined]
99 + self.sender_socket = self.context.socket(zmq.constants.PUSH)
100 + self.receiver_socket = self.context.socket(zmq.constants.PULL)
101 + self.sender_ack = self.context.socket(zmq.constants.PULL)
102 + self.receiver_ack = self.context.socket(zmq.constants.PUSH)
103 +
104 + self.buffer_cleaner = ThreadPoolExecutor(max_workers=1)
105 + self._setup_metadata_sockets(kv_rank, prefill_host, base_prefill_port,
106 + decode_host, base_decode_port)
107 +
108 + def _setup_metadata_sockets(self, kv_rank: int, p_host: str, p_port: str,
109 + d_host: str, d_port: str) -> None:
110 + """Set up ZeroMQ sockets for sending and receiving data."""
111 + # Offsets < 8 are left for initialization in case tp and pp are enabled
112 + p_rank_offset = int(p_port) + 8 + self.local_rank * 2
113 + d_rank_offset = int(d_port) + 8 + self.local_rank * 2
114 + if kv_rank == 0:
115 + self.sender_socket.bind(f"tcp://*:{p_rank_offset + 1}")
116 + self.receiver_socket.connect(f"tcp://{d_host}:{d_rank_offset + 1}")
117 + self.sender_ack.connect(f"tcp://{d_host}:{d_rank_offset + 2}")
118 + self.receiver_ack.bind(f"tcp://*:{p_rank_offset + 2}")
119 + else:
120 + self.receiver_socket.connect(f"tcp://{p_host}:{p_rank_offset + 1}")
121 + self.sender_socket.bind(f"tcp://*:{d_rank_offset + 1}")
122 + self.receiver_ack.bind(f"tcp://*:{d_rank_offset + 2}")
123 + self.sender_ack.connect(f"tcp://{p_host}:{p_rank_offset + 2}")
124 +
125 + def initialize(self, local_hostname: str, metadata_server: str,
126 + protocol: str, device_name: str,
127 + metadata_backend: Union[str, None]) -> None:
128 + """Initialize the mooncake instance."""
129 + if metadata_backend is None:
130 + self.engine.initialize(local_hostname, metadata_server, protocol,
131 + device_name)
132 + else:
133 + supported_backend = ["etcd", "redis"]
134 + metadata_backend = metadata_backend.lower()
135 + if metadata_backend not in supported_backend:
136 + raise ValueError(
137 + "Mooncake Configuration error. `metadata_backend`"
138 + f"should be one of {supported_backend}.")
139 +
140 + self.engine.initializeExt(local_hostname, metadata_server,
141 + protocol, device_name, metadata_backend)
142 +
143 + def allocate_managed_buffer(self, length: int) -> int:
144 + """Allocate a managed buffer of the specified length."""
145 + ret = self.engine.allocateManagedBuffer(length)
146 + if ret <= 0:
147 + logger.error("Allocation Return Error")
148 + raise Exception("Allocation Return Error")
149 + return ret
150 +
151 + def free_managed_buffer(self, buffer: int, length: int) -> int:
152 + """Free a previously allocated managed buffer."""
153 + return self.engine.freeManagedBuffer(buffer, length)
154 +
155 + def transfer_sync(self, buffer: int, peer_buffer_address: int,
156 + length: int) -> int:
157 + """Synchronously transfer data to the specified address."""
158 + ret = self.engine.transferSync(self.remote_url, buffer,
159 + peer_buffer_address, length)
160 + if ret < 0:
161 + logger.error("Transfer Return Error")
162 + raise Exception("Transfer Return Error")
163 + return ret
164 +
165 + def write_bytes_to_buffer(self, buffer: int, user_data: bytes,
166 + length: int) -> int:
167 + """Write bytes to the allocated buffer."""
168 + return self.engine.writeBytesToBuffer(buffer, user_data, length)
169 +
170 + def read_bytes_from_buffer(self, buffer: int, length: int) -> bytes:
171 + """Read bytes from the allocated buffer."""
172 + return self.engine.readBytesFromBuffer(buffer, length)
173 +
174 + def wait_for_ack(self, src_ptr: int, length: int) -> None:
175 + """Asynchronously wait for ACK from the receiver."""
176 + ack = self.sender_ack.recv_pyobj()
177 + if ack != b'ACK':
178 + logger.error("Failed to receive ACK from the receiver")
179 +
180 + self.free_managed_buffer(src_ptr, length)
181 +
182 + def send_bytes(self, user_data: bytes) -> None:
183 + """Send bytes to the remote process."""
184 + length = len(user_data)
185 + src_ptr = self.allocate_managed_buffer(length)
186 + self.write_bytes_to_buffer(src_ptr, user_data, length)
187 + self.sender_socket.send_pyobj((src_ptr, length))
188 + self.buffer_cleaner.submit(self.wait_for_ack, src_ptr, length)
189 +
190 + def recv_bytes(self) -> bytes:
191 + """Receive bytes from the remote process."""
192 + src_ptr, length = self.receiver_socket.recv_pyobj()
193 + dst_ptr = self.allocate_managed_buffer(length)
194 + self.transfer_sync(dst_ptr, src_ptr, length)
195 + ret = self.read_bytes_from_buffer(dst_ptr, length)
196 +
197 + # Buffer cleanup
198 + self.receiver_ack.send_pyobj(b'ACK')
199 + self.free_managed_buffer(dst_ptr, length)
200 +
201 + return ret
202 +
203 +
204 +class MooncakePipe(KVPipeBase):
205 + """MooncakeTransferEngine based Pipe implementation."""
206 +
207 + def __init__(self,
208 + local_rank: int,
209 + config: KVTransferConfig,
210 + device: Optional[str] = None):
211 + """Initialize the mooncake pipe and set related parameters."""
212 + self.config = config
213 + self.local_rank = local_rank
214 + self.kv_rank = self.config.kv_rank
215 + if device is None:
216 + self.device = self._select_device(self.config.kv_buffer_device)
217 + else:
218 + self.device = self._select_device(device)
219 +
220 + self.transfer_engine = MooncakeTransferEngine(self.kv_rank,
221 + self.local_rank)
222 + self.transport_thread: Optional[ThreadPoolExecutor] = None
223 + self.none_tensor = torch.tensor([NONE_INT], device=self.device)
224 +
225 + def _select_device(self, device: str) -> torch.device:
226 + """Select available device (CUDA or CPU)."""
227 + logger.info("Selecting device: %s", device)
228 + if device == "cuda":
229 + return torch.device(f"cuda:{self.local_rank}")
230 + else:
231 + return torch.device("cpu")
232 +
233 + def tensor_hash(self, tensor: torch.Tensor) -> int:
234 + """Calculate the hash value of the tensor."""
235 + return hash(tensor.data_ptr())
236 +
237 + def _send_impl(self, tensor: torch.Tensor) -> None:
238 + """Implement the tensor sending logic."""
239 + value_bytes = pickle.dumps(tensor)
240 + self.transfer_engine.send_bytes(value_bytes)
241 +
242 + def _recv_impl(self) -> torch.Tensor:
243 + """Implement the tensor receiving logic."""
244 + data = self.transfer_engine.recv_bytes()
245 + return pickle.loads(data)
246 +
247 + def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
248 + """Send tensor to the target process."""
249 + if self.transport_thread is None:
250 + self.transport_thread = ThreadPoolExecutor(max_workers=1)
251 + tensor = tensor if tensor is not None else self.none_tensor
252 + assert (len(tensor.shape) > 0)
253 + self.transport_thread.submit(self._send_impl, tensor)
254 +
255 + def recv_tensor(self) -> Optional[torch.Tensor]:
256 + """Receive tensor from other processes."""
257 + if self.transport_thread is None:
258 + self.transport_thread = ThreadPoolExecutor(max_workers=1)
259 + tensor = self.transport_thread.submit(self._recv_impl).result()
260 + if tensor.numel() == 1 and tensor.item() == NONE_INT:
261 + return None
262 + else:
263 + return tensor
264 +
265 + def close(self) -> None:
266 + """Cleanup logic when closing the pipe."""
267 + self.transfer_engine.sender_socket.close()
268 + self.transfer_engine.receiver_socket.close()
269 + self.transfer_engine.sender_ack.close()
270 + self.transfer_engine.receiver_ack.close()
271 + self.transfer_engine.context.term() # Terminate the ZMQ context
272 + logger.info("Closed the transfer engine and cleaned up resources.")