A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Core] Support disaggregated prefill with Mooncake Transfer Engine (#10884)
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
shangmingc committed
Dec 16, 2024 at 05:28 UTC
d263bd9df7b2f5586910e5d006a11ff11ba7c310
4 files changed
+352
-31
vllm/config.py
+4
-3
@@ -2171,13 +2171,14 @@ class KVTransferConfig(BaseModel):
2171
return KVTransferConfig.model_validate_json(cli_value)
2172
2173
def model_post_init(self, __context: Any) -> None:
2174
+ supported_kv_connector = ["PyNcclConnector", "MooncakeConnector"]
2175
if all([
2175
- self.kv_connector is not None,
2176
- self.kv_connector != "PyNcclConnector"
2176
+ self.kv_connector is not None, self.kv_connector
2177
+ not in supported_kv_connector
2178
]):
2179
raise ValueError(f"Unsupported kv_connector: {self.kv_connector}. "
2180
f"Supported connectors are "
2180
- f"`PyNcclConnector`.")
2181
+ f"{supported_kv_connector}.")
2182
2183
if self.kv_role is not None and self.kv_role not in [
2184
"kv_producer", "kv_consumer", "kv_both"
vllm/distributed/kv_transfer/kv_connector/factory.py
+2
-1
@@ -11,7 +11,8 @@ class KVConnectorFactory:
11
@staticmethod
12
def create_connector(rank: int, local_rank: int,
13
config: "VllmConfig") -> KVConnectorBase:
14
- if config.kv_transfer_config.kv_connector == 'PyNcclConnector':
14
+ supported_kv_connector = ["PyNcclConnector", "MooncakeConnector"]
15
+ if config.kv_transfer_config.kv_connector in supported_kv_connector:
16
from .simple_connector import SimpleConnector
17
return SimpleConnector(rank, local_rank, config)
18
else:
vllm/distributed/kv_transfer/kv_connector/simple_connector.py
+74
-27
@@ -2,7 +2,8 @@
2
Simple KV Cache Connector for Distributed Machine Learning Inference
3
4
The SimpleConnector transfers KV caches between prefill vLLM worker (KV cache
5
-producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe.
5
+producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe or
6
+MooncakePipe.
7
8
But the logic can be extended to support other pipe and lookup buffer.
9
"""
@@ -15,7 +16,6 @@ from vllm.config import VllmConfig
16
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase
17
from vllm.distributed.kv_transfer.kv_lookup_buffer.simple_buffer import (
18
SimpleBuffer)
18
-from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
19
from vllm.logger import init_logger
20
from vllm.sequence import IntermediateTensors
21
@@ -36,7 +36,27 @@ class SimpleConnector(KVConnectorBase):
36
37
self.config = config.kv_transfer_config
38
39
- logger.info("Initializing PyNcclConfig under kv_transfer_config %s",
39
+ if self.config.kv_connector == "PyNcclConnector":
40
+ from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import (
41
+ PyNcclPipe)
42
+ logger.info(
43
+ "Initializing PyNcclConfig under kv_transfer_config %s",
44
+ self.config)
45
+ elif self.config.kv_connector == "MooncakeConnector":
46
+ # Check if MOONCAKE_CONFIG_PATH is set
47
+ import os
48
+ use_mooncake_distributed_pipe = os.getenv(
49
+ 'MOONCAKE_CONFIG_PATH') is not None
50
+
51
+ if not use_mooncake_distributed_pipe:
52
+ raise ValueError(
53
+ "To use MooncakeConnector, you need to pass the ENV: "
54
+ "'MOONCAKE_CONFIG_PATH=/path/to/mooncake_config.json'.")
55
+ else:
56
+ from vllm.distributed.kv_transfer.kv_pipe.mooncake_pipe import ( # noqa: E501
57
+ MooncakePipe)
58
+ logger.info(
59
+ "Initializing MooncakeConfig under kv_transfer_config %s",
60
self.config)
61
62
self.lookup_buffer_size = self.config.kv_buffer_size
@@ -44,6 +64,11 @@ class SimpleConnector(KVConnectorBase):
64
self.producer_buffer: Optional[SimpleBuffer] = None
65
self.consumer_buffer: Optional[SimpleBuffer] = None
66
67
+ self.producer_data_pipe: Union[PyNcclPipe, MooncakePipe]
68
+ self.consumer_data_pipe: Union[PyNcclPipe, MooncakePipe]
69
+ self.producer_signal_pipe: Union[PyNcclPipe, MooncakePipe]
70
+ self.consumer_signal_pipe: Union[PyNcclPipe, MooncakePipe]
71
+
72
# 2 pipes for every rank in the world
73
port_offset_base = 2 * rank
74
@@ -51,17 +76,26 @@ class SimpleConnector(KVConnectorBase):
76
# and the decode vLLM only uses recv pipe
77
if self.config.is_kv_producer:
78
54
- self.producer_data_pipe = PyNcclPipe(
55
- local_rank=local_rank,
56
- config=self.config,
57
- port_offset=port_offset_base,
58
- )
59
- self.producer_signal_pipe = PyNcclPipe(
60
- local_rank=local_rank,
61
- config=self.config,
62
- port_offset=port_offset_base + 1,
63
- device="cpu",
64
- )
79
+ if self.config.kv_connector == "PyNcclConnector":
80
+ self.producer_data_pipe = PyNcclPipe(
81
+ local_rank=local_rank,
82
+ config=self.config,
83
+ port_offset=port_offset_base,
84
+ )
85
+ self.producer_signal_pipe = PyNcclPipe(
86
+ local_rank=local_rank,
87
+ config=self.config,
88
+ port_offset=port_offset_base + 1,
89
+ device="cpu",
90
+ )
91
+ elif self.config.kv_connector == "MooncakeConnector":
92
+ self.producer_data_pipe = MooncakePipe(
93
+ local_rank=local_rank,
94
+ config=self.config,
95
+ )
96
+ # We only need to initialize MooncakePipe once
97
+ self.producer_signal_pipe = self.producer_data_pipe
98
+
99
self.producer_buffer = SimpleBuffer(self.producer_signal_pipe,
100
self.producer_data_pipe,
101
self.config.kv_buffer_size)
@@ -70,17 +104,25 @@ class SimpleConnector(KVConnectorBase):
104
105
# the current vLLM instance is KV consumer, so it needs to connect
106
# its recv pipe to the send pipe of KV producder
73
- self.consumer_data_pipe = PyNcclPipe(
74
- local_rank=local_rank,
75
- config=self.config,
76
- port_offset=port_offset_base,
77
- )
78
- self.consumer_signal_pipe = PyNcclPipe(
79
- local_rank=local_rank,
80
- config=self.config,
81
- port_offset=port_offset_base + 1,
82
- device="cpu",
83
- )
107
+ if self.config.kv_connector == "PyNcclConnector":
108
+ self.consumer_data_pipe = PyNcclPipe(
109
+ local_rank=local_rank,
110
+ config=self.config,
111
+ port_offset=port_offset_base,
112
+ )
113
+ self.consumer_signal_pipe = PyNcclPipe(
114
+ local_rank=local_rank,
115
+ config=self.config,
116
+ port_offset=port_offset_base + 1,
117
+ device="cpu",
118
+ )
119
+ elif self.config.kv_connector == "MooncakeConnector":
120
+ self.consumer_data_pipe = MooncakePipe(
121
+ local_rank=local_rank,
122
+ config=self.config,
123
+ )
124
+ self.consumer_signal_pipe = self.consumer_data_pipe
125
+
126
self.consumer_buffer = SimpleBuffer(
127
self.consumer_signal_pipe,
128
self.consumer_data_pipe,
@@ -260,6 +302,11 @@ class SimpleConnector(KVConnectorBase):
302
303
def close(self):
304
self.producer_data_pipe.close()
263
- self.producer_signal_pipe.close()
305
self.consumer_data_pipe.close()
265
- self.consumer_signal_pipe.close()
306
+ if self.config.kv_connector == "PyNcclConnector":
307
+ self.producer_signal_pipe.close()
308
+ self.consumer_signal_pipe.close()
309
+ elif self.config.kv_connector == "MooncakeConnector":
310
+ # MooncakePipe reuses data_pipe for signal_pipe, so we only have to
311
+ # close the data_pipe.
312
+ pass
vllm/distributed/kv_transfer/kv_pipe/mooncake_pipe.py
new
+272
@@ -0,0 +1,272 @@
1
+import json
2
+import os
3
+import pickle
4
+from concurrent.futures import ThreadPoolExecutor
5
+from dataclasses import dataclass
6
+from typing import Optional, Union
7
+
8
+import torch
9
+import zmq
10
+
11
+from vllm.config import KVTransferConfig
12
+from vllm.distributed.kv_transfer.kv_pipe.base import KVPipeBase
13
+from vllm.logger import init_logger
14
+
15
+logger = init_logger(__name__)
16
+NONE_INT = -150886311
17
+
18
+
19
+@dataclass
20
+class MooncakeTransferEngineConfig:
21
+ prefill_url: str
22
+ decode_url: str
23
+ metadata_backend: Union[str, None]
24
+ metadata_server: str
25
+ protocol: str
26
+ device_name: str
27
+
28
+ @staticmethod
29
+ def from_file(file_path: str) -> 'MooncakeTransferEngineConfig':
30
+ """Load the config from a JSON file."""
31
+ with open(file_path) as fin:
32
+ config = json.load(fin)
33
+ return MooncakeTransferEngineConfig(
34
+ prefill_url=config.get("prefill_url"),
35
+ decode_url=config.get("decode_url"),
36
+ metadata_backend=config.get("metadata_backend", None),
37
+ metadata_server=config.get("metadata_server"),
38
+ protocol=config.get("protocol", "tcp"),
39
+ device_name=config.get("device_name", ""),
40
+ )
41
+
42
+ @staticmethod
43
+ def load_from_env() -> 'MooncakeTransferEngineConfig':
44
+ """Load config from a file specified in the environment variable."""
45
+ config_file_path = os.getenv('MOONCAKE_CONFIG_PATH')
46
+ if config_file_path is None:
47
+ raise ValueError(
48
+ "The environment variable 'MOONCAKE_CONFIG_PATH' is not set.")
49
+ return MooncakeTransferEngineConfig.from_file(config_file_path)
50
+
51
+
52
+class MooncakeTransferEngine:
53
+ """Handles the transfer of data using mooncake_vllm_adaptor and ZeroMQ."""
54
+
55
+ def __init__(self, kv_rank: int, local_rank: int):
56
+ try:
57
+ import mooncake_vllm_adaptor as mva
58
+ except ImportError as e:
59
+ raise ImportError(
60
+ "Please install mooncake by following the instructions at "
61
+ "https://github.com/kvcache-ai/Mooncake/blob/main/doc/en/build.md " # noqa: E501
62
+ "to run vLLM with MooncakeConnector.") from e
63
+
64
+ self.engine = mva.mooncake_vllm_adaptor()
65
+ self.local_rank = local_rank
66
+
67
+ try:
68
+ self.config = MooncakeTransferEngineConfig.load_from_env()
69
+ logger.info("Mooncake Configuration loaded successfully.")
70
+ except ValueError as e:
71
+ logger.error(e)
72
+ raise
73
+ except Exception as exc:
74
+ logger.error(
75
+ "An error occurred while loading the configuration: %s", exc)
76
+ raise
77
+ prefill_host, base_prefill_port = self.config.prefill_url.split(':')
78
+ decode_host, base_decode_port = self.config.decode_url.split(':')
79
+
80
+ # Avoid ports conflict when running prefill and decode on the same node
81
+ if prefill_host == decode_host and \
82
+ base_prefill_port == base_decode_port:
83
+ base_decode_port = str(int(base_decode_port) + 100)
84
+
85
+ prefill_port = int(base_prefill_port) + self.local_rank
86
+ decode_port = int(base_decode_port) + self.local_rank
87
+ self.prefill_url = ':'.join([prefill_host, str(prefill_port)])
88
+ self.decode_url = ':'.join([decode_host, str(decode_port)])
89
+
90
+ self.initialize(self.prefill_url if kv_rank == 0 else self.decode_url,
91
+ self.config.metadata_server, self.config.protocol,
92
+ self.config.device_name, self.config.metadata_backend)
93
+
94
+ self.remote_url = (self.decode_url
95
+ if kv_rank == 0 else self.prefill_url)
96
+
97
+ # Initialize ZeroMQ context and sockets
98
+ self.context = zmq.Context() # type: ignore[attr-defined]
99
+ self.sender_socket = self.context.socket(zmq.constants.PUSH)
100
+ self.receiver_socket = self.context.socket(zmq.constants.PULL)
101
+ self.sender_ack = self.context.socket(zmq.constants.PULL)
102
+ self.receiver_ack = self.context.socket(zmq.constants.PUSH)
103
+
104
+ self.buffer_cleaner = ThreadPoolExecutor(max_workers=1)
105
+ self._setup_metadata_sockets(kv_rank, prefill_host, base_prefill_port,
106
+ decode_host, base_decode_port)
107
+
108
+ def _setup_metadata_sockets(self, kv_rank: int, p_host: str, p_port: str,
109
+ d_host: str, d_port: str) -> None:
110
+ """Set up ZeroMQ sockets for sending and receiving data."""
111
+ # Offsets < 8 are left for initialization in case tp and pp are enabled
112
+ p_rank_offset = int(p_port) + 8 + self.local_rank * 2
113
+ d_rank_offset = int(d_port) + 8 + self.local_rank * 2
114
+ if kv_rank == 0:
115
+ self.sender_socket.bind(f"tcp://*:{p_rank_offset + 1}")
116
+ self.receiver_socket.connect(f"tcp://{d_host}:{d_rank_offset + 1}")
117
+ self.sender_ack.connect(f"tcp://{d_host}:{d_rank_offset + 2}")
118
+ self.receiver_ack.bind(f"tcp://*:{p_rank_offset + 2}")
119
+ else:
120
+ self.receiver_socket.connect(f"tcp://{p_host}:{p_rank_offset + 1}")
121
+ self.sender_socket.bind(f"tcp://*:{d_rank_offset + 1}")
122
+ self.receiver_ack.bind(f"tcp://*:{d_rank_offset + 2}")
123
+ self.sender_ack.connect(f"tcp://{p_host}:{p_rank_offset + 2}")
124
+
125
+ def initialize(self, local_hostname: str, metadata_server: str,
126
+ protocol: str, device_name: str,
127
+ metadata_backend: Union[str, None]) -> None:
128
+ """Initialize the mooncake instance."""
129
+ if metadata_backend is None:
130
+ self.engine.initialize(local_hostname, metadata_server, protocol,
131
+ device_name)
132
+ else:
133
+ supported_backend = ["etcd", "redis"]
134
+ metadata_backend = metadata_backend.lower()
135
+ if metadata_backend not in supported_backend:
136
+ raise ValueError(
137
+ "Mooncake Configuration error. `metadata_backend`"
138
+ f"should be one of {supported_backend}.")
139
+
140
+ self.engine.initializeExt(local_hostname, metadata_server,
141
+ protocol, device_name, metadata_backend)
142
+
143
+ def allocate_managed_buffer(self, length: int) -> int:
144
+ """Allocate a managed buffer of the specified length."""
145
+ ret = self.engine.allocateManagedBuffer(length)
146
+ if ret <= 0:
147
+ logger.error("Allocation Return Error")
148
+ raise Exception("Allocation Return Error")
149
+ return ret
150
+
151
+ def free_managed_buffer(self, buffer: int, length: int) -> int:
152
+ """Free a previously allocated managed buffer."""
153
+ return self.engine.freeManagedBuffer(buffer, length)
154
+
155
+ def transfer_sync(self, buffer: int, peer_buffer_address: int,
156
+ length: int) -> int:
157
+ """Synchronously transfer data to the specified address."""
158
+ ret = self.engine.transferSync(self.remote_url, buffer,
159
+ peer_buffer_address, length)
160
+ if ret < 0:
161
+ logger.error("Transfer Return Error")
162
+ raise Exception("Transfer Return Error")
163
+ return ret
164
+
165
+ def write_bytes_to_buffer(self, buffer: int, user_data: bytes,
166
+ length: int) -> int:
167
+ """Write bytes to the allocated buffer."""
168
+ return self.engine.writeBytesToBuffer(buffer, user_data, length)
169
+
170
+ def read_bytes_from_buffer(self, buffer: int, length: int) -> bytes:
171
+ """Read bytes from the allocated buffer."""
172
+ return self.engine.readBytesFromBuffer(buffer, length)
173
+
174
+ def wait_for_ack(self, src_ptr: int, length: int) -> None:
175
+ """Asynchronously wait for ACK from the receiver."""
176
+ ack = self.sender_ack.recv_pyobj()
177
+ if ack != b'ACK':
178
+ logger.error("Failed to receive ACK from the receiver")
179
+
180
+ self.free_managed_buffer(src_ptr, length)
181
+
182
+ def send_bytes(self, user_data: bytes) -> None:
183
+ """Send bytes to the remote process."""
184
+ length = len(user_data)
185
+ src_ptr = self.allocate_managed_buffer(length)
186
+ self.write_bytes_to_buffer(src_ptr, user_data, length)
187
+ self.sender_socket.send_pyobj((src_ptr, length))
188
+ self.buffer_cleaner.submit(self.wait_for_ack, src_ptr, length)
189
+
190
+ def recv_bytes(self) -> bytes:
191
+ """Receive bytes from the remote process."""
192
+ src_ptr, length = self.receiver_socket.recv_pyobj()
193
+ dst_ptr = self.allocate_managed_buffer(length)
194
+ self.transfer_sync(dst_ptr, src_ptr, length)
195
+ ret = self.read_bytes_from_buffer(dst_ptr, length)
196
+
197
+ # Buffer cleanup
198
+ self.receiver_ack.send_pyobj(b'ACK')
199
+ self.free_managed_buffer(dst_ptr, length)
200
+
201
+ return ret
202
+
203
+
204
+class MooncakePipe(KVPipeBase):
205
+ """MooncakeTransferEngine based Pipe implementation."""
206
+
207
+ def __init__(self,
208
+ local_rank: int,
209
+ config: KVTransferConfig,
210
+ device: Optional[str] = None):
211
+ """Initialize the mooncake pipe and set related parameters."""
212
+ self.config = config
213
+ self.local_rank = local_rank
214
+ self.kv_rank = self.config.kv_rank
215
+ if device is None:
216
+ self.device = self._select_device(self.config.kv_buffer_device)
217
+ else:
218
+ self.device = self._select_device(device)
219
+
220
+ self.transfer_engine = MooncakeTransferEngine(self.kv_rank,
221
+ self.local_rank)
222
+ self.transport_thread: Optional[ThreadPoolExecutor] = None
223
+ self.none_tensor = torch.tensor([NONE_INT], device=self.device)
224
+
225
+ def _select_device(self, device: str) -> torch.device:
226
+ """Select available device (CUDA or CPU)."""
227
+ logger.info("Selecting device: %s", device)
228
+ if device == "cuda":
229
+ return torch.device(f"cuda:{self.local_rank}")
230
+ else:
231
+ return torch.device("cpu")
232
+
233
+ def tensor_hash(self, tensor: torch.Tensor) -> int:
234
+ """Calculate the hash value of the tensor."""
235
+ return hash(tensor.data_ptr())
236
+
237
+ def _send_impl(self, tensor: torch.Tensor) -> None:
238
+ """Implement the tensor sending logic."""
239
+ value_bytes = pickle.dumps(tensor)
240
+ self.transfer_engine.send_bytes(value_bytes)
241
+
242
+ def _recv_impl(self) -> torch.Tensor:
243
+ """Implement the tensor receiving logic."""
244
+ data = self.transfer_engine.recv_bytes()
245
+ return pickle.loads(data)
246
+
247
+ def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
248
+ """Send tensor to the target process."""
249
+ if self.transport_thread is None:
250
+ self.transport_thread = ThreadPoolExecutor(max_workers=1)
251
+ tensor = tensor if tensor is not None else self.none_tensor
252
+ assert (len(tensor.shape) > 0)
253
+ self.transport_thread.submit(self._send_impl, tensor)
254
+
255
+ def recv_tensor(self) -> Optional[torch.Tensor]:
256
+ """Receive tensor from other processes."""
257
+ if self.transport_thread is None:
258
+ self.transport_thread = ThreadPoolExecutor(max_workers=1)
259
+ tensor = self.transport_thread.submit(self._recv_impl).result()
260
+ if tensor.numel() == 1 and tensor.item() == NONE_INT:
261
+ return None
262
+ else:
263
+ return tensor
264
+
265
+ def close(self) -> None:
266
+ """Cleanup logic when closing the pipe."""
267
+ self.transfer_engine.sender_socket.close()
268
+ self.transfer_engine.receiver_socket.close()
269
+ self.transfer_engine.sender_ack.close()
270
+ self.transfer_engine.receiver_ack.close()
271
+ self.transfer_engine.context.term() # Terminate the ZMQ context
272
+ logger.info("Closed the transfer engine and cleaned up resources.")