| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """NCCL-based (dense) weight transfer engine.""" |
| 4 | |
| 5 | from collections.abc import Iterator |
| 6 | from concurrent.futures import ThreadPoolExecutor |
| 7 | from dataclasses import asdict, dataclass |
| 8 | from typing import TYPE_CHECKING, ClassVar |
| 9 | |
| 10 | import torch |
| 11 | from typing_extensions import Self |
| 12 | |
| 13 | if TYPE_CHECKING: |
| 14 | from vllm.config import VllmConfig |
| 15 | from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator |
| 16 | |
| 17 | from vllm.config.weight_transfer import WeightTransferConfig |
| 18 | from vllm.distributed.weight_transfer.base import ( |
| 19 | ParamMeta, |
| 20 | TrainerInitInfo, |
| 21 | TrainerWeightTransferEngine, |
| 22 | VLLMWeightSyncClient, |
| 23 | WeightSource, |
| 24 | WeightTransferEngine, |
| 25 | WeightTransferUpdateInfo, |
| 26 | ) |
| 27 | from vllm.distributed.weight_transfer.nccl_common import ( |
| 28 | NCCLWeightTransferInitInfo, |
| 29 | worker_init_payload, |
| 30 | worker_init_process_group, |
| 31 | ) |
| 32 | from vllm.distributed.weight_transfer.nccl_common import ( |
| 33 | trainer_init as open_trainer_endpoint, |
| 34 | ) |
| 35 | from vllm.distributed.weight_transfer.packed_tensor import ( |
| 36 | DEFAULT_PACKED_BUFFER_SIZE_BYTES, |
| 37 | DEFAULT_PACKED_NUM_BUFFERS, |
| 38 | packed_nccl_broadcast_consumer, |
| 39 | packed_nccl_broadcast_producer, |
| 40 | ) |
| 41 | |
| 42 | # NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical |
| 43 | # home is nccl_common, shared with the sparse backend. |
| 44 | __all__ = [ |
| 45 | "NCCLWeightTransferInitInfo", |
| 46 | "NCCLTrainerInitInfo", |
| 47 | "NCCLWeightTransferUpdateInfo", |
| 48 | "NCCLWeightTransferEngine", |
| 49 | "NCCLTrainerWeightTransferEngine", |
| 50 | ] |
| 51 | |
| 52 | |
| 53 | @dataclass |
| 54 | class NCCLTrainerInitInfo(TrainerInitInfo): |
| 55 | """Trainer-side init info for the dense NCCL weight transfer backend. |
| 56 | |
| 57 | The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`. |
| 58 | `world_size` is the full trainer+worker NCCL group size. `rank` (from |
| 59 | `TrainerInitInfo`) identifies this trainer process; rank 0 is the sender. |
| 60 | |
| 61 | The trainer joins over a TCPStore rendezvous (`master_address` + |
| 62 | `master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore |
| 63 | mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a |
| 64 | `nccl_unique_id_b64` payload straight to the inference workers' |
| 65 | `init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than |
| 66 | going through this engine. |
| 67 | |
| 68 | `packed` / buffer sizes are the transfer's wire params. The trainer |
| 69 | propagates them to the worker at `trainer_init` so the two sides cannot |
| 70 | disagree. Note this defaults to packed, unlike the worker-side |
| 71 | `NCCLWeightTransferInitInfo`, whose default only applies when no trainer |
| 72 | ships a value. `backend` is the factory dispatch key.""" |
| 73 | |
| 74 | backend: ClassVar[str] = "nccl" |
| 75 | |
| 76 | master_address: str |
| 77 | master_port: int |
| 78 | world_size: int |
| 79 | packed: bool = True |
| 80 | packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES |
| 81 | packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS |
| 82 | |
| 83 | |
| 84 | @dataclass |
| 85 | class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo): |
| 86 | """Per-round update info for the dense NCCL weight transfer backend. |
| 87 | |
| 88 | Whether the transfer is packed (and the buffer geometry) is a must-agree |
| 89 | wire param carried on the init info (`NCCLTrainerInitInfo` / |
| 90 | `NCCLWeightTransferInitInfo`), not here; this carries only the per-round |
| 91 | parameter metadata. |
| 92 | """ |
| 93 | |
| 94 | names: list[str] |
| 95 | dtype_names: list[str] |
| 96 | shapes: list[list[int]] |
| 97 | |
| 98 | def __post_init__(self): |
| 99 | """Validate that all lists have the same length.""" |
| 100 | num_params = len(self.names) |
| 101 | if len(self.dtype_names) != num_params: |
| 102 | raise ValueError( |
| 103 | f"`dtype_names` should be of the same size as `names`: " |
| 104 | f"got {len(self.dtype_names)} and {len(self.names)}" |
| 105 | ) |
| 106 | if len(self.shapes) != num_params: |
| 107 | raise ValueError( |
| 108 | f"`shapes` should be of the same size as `names`: " |
| 109 | f"got {len(self.shapes)} and {len(self.names)}" |
| 110 | ) |
| 111 | |
| 112 | |
| 113 | class NCCLWeightTransferEngine( |
| 114 | WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo] |
| 115 | ): |
| 116 | """Weight transfer engine using NCCL for communication between trainer and workers. |
| 117 | |
| 118 | This implementation uses NCCL broadcast operations to transfer dense |
| 119 | checkpoint-format weights from the trainer (rank 0) to all inference workers |
| 120 | in a process group. Received weights are loaded via the model's |
| 121 | `load_weights` using the layerwise reload lifecycle. |
| 122 | """ |
| 123 | |
| 124 | # Define backend-specific dataclass types |
| 125 | init_info_cls = NCCLWeightTransferInitInfo |
| 126 | update_info_cls = NCCLWeightTransferUpdateInfo |
| 127 | |
| 128 | def __init__( |
| 129 | self, |
| 130 | config: WeightTransferConfig, |
| 131 | vllm_config: "VllmConfig", |
| 132 | device: torch.device, |
| 133 | model: torch.nn.Module, |
| 134 | ) -> None: |
| 135 | super().__init__(config, vllm_config, device, model) |
| 136 | self.model_update_group: PyNcclCommunicator | None = None |
| 137 | # Set from the trainer-supplied init info at the handshake; defaults are |
| 138 | # only for the (unreachable) receive-before-init case. |
| 139 | self.packed = False |
| 140 | self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES |
| 141 | self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS |
| 142 | |
| 143 | def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None: |
| 144 | """Initialize NCCL process group with the trainer and record the |
| 145 | trainer-supplied wire params so the worker decodes exactly as the |
| 146 | trainer encodes. |
| 147 | |
| 148 | Args: |
| 149 | init_info: NCCL initialization info containing master address, port, |
| 150 | rank offset, world size, and the packed wire params |
| 151 | |
| 152 | """ |
| 153 | self.packed = init_info.packed |
| 154 | self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes |
| 155 | self.packed_num_buffers = init_info.packed_num_buffers |
| 156 | self.model_update_group = worker_init_process_group( |
| 157 | init_info, self.parallel_config |
| 158 | ) |
| 159 | |
| 160 | def start_weight_update(self) -> None: |
| 161 | """Initialize layerwise reloading for the incoming checkpoint weights.""" |
| 162 | from vllm.model_executor.model_loader.reload import ( |
| 163 | initialize_layerwise_reload, |
| 164 | ) |
| 165 | |
| 166 | initialize_layerwise_reload(self.model) |
| 167 | |
| 168 | def finish_weight_update(self) -> None: |
| 169 | """Finalize layerwise reloading after all weights have been received.""" |
| 170 | from vllm.model_executor.model_loader.reload import ( |
| 171 | finalize_layerwise_reload, |
| 172 | ) |
| 173 | |
| 174 | finalize_layerwise_reload(self.model, self.model_config) |
| 175 | |
| 176 | def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None: |
| 177 | """Receive weights from trainer via NCCL broadcast. |
| 178 | |
| 179 | Whether to use packed broadcasting (and the buffer geometry) is read |
| 180 | from `self.packed` / `self.packed_*`, set at the init handshake from the |
| 181 | trainer's init info, so it is guaranteed to match how the trainer |
| 182 | encoded. |
| 183 | |
| 184 | Args: |
| 185 | update_info: NCCL update info containing parameter names, dtypes, |
| 186 | and shapes |
| 187 | |
| 188 | """ |
| 189 | if self.model_update_group is None: |
| 190 | raise RuntimeError( |
| 191 | "NCCL weight transfer not initialized. " |
| 192 | "Call init_transfer_engine() first." |
| 193 | ) |
| 194 | |
| 195 | from vllm.model_executor.model_loader.mtp_validation import ( |
| 196 | disable_mtp_completeness_check, |
| 197 | ) |
| 198 | |
| 199 | with disable_mtp_completeness_check(): |
| 200 | if self.packed: |
| 201 | # Build iterator of (name, (shape, dtype)) from update_info |
| 202 | def state_dict_info_iterator(): |
| 203 | for name, dtype_name, shape in zip( |
| 204 | update_info.names, update_info.dtype_names, update_info.shapes |
| 205 | ): |
| 206 | dtype = getattr(torch, dtype_name) |
| 207 | yield (name, (shape, dtype)) |
| 208 | |
| 209 | packed_nccl_broadcast_consumer( |
| 210 | iterator=state_dict_info_iterator(), |
| 211 | group=self.model_update_group, |
| 212 | src=0, |
| 213 | post_unpack_func=self.model.load_weights, |
| 214 | buffer_size_bytes=self.packed_buffer_size_bytes, |
| 215 | num_buffers=self.packed_num_buffers, |
| 216 | device=self.device, |
| 217 | ) |
| 218 | else: |
| 219 | # Use simple one-by-one broadcasting |
| 220 | for name, dtype_name, shape in zip( |
| 221 | update_info.names, update_info.dtype_names, update_info.shapes |
| 222 | ): |
| 223 | dtype = getattr(torch, dtype_name) |
| 224 | weight = torch.empty(shape, dtype=dtype, device=self.device) |
| 225 | self.model_update_group.broadcast( |
| 226 | weight, src=0, stream=torch.cuda.current_stream() |
| 227 | ) |
| 228 | self.model.load_weights([(name, weight)]) |
| 229 | del weight |
| 230 | |
| 231 | def shutdown(self) -> None: |
| 232 | if self.model_update_group is not None: |
| 233 | # Clean up the communicator by removing the reference |
| 234 | self.model_update_group = None |
| 235 | |
| 236 | |
| 237 | class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]): |
| 238 | """Trainer-side NCCL weight transfer engine. |
| 239 | |
| 240 | On the sender (rank 0) holds the NCCL communicator and drives the full |
| 241 | update round trip: it runs the inference-side `update_weights` concurrently |
| 242 | with the trainer-side broadcast (both rendezvous inside the same NCCL |
| 243 | calls), then finishes the update. Non-sender trainer ranks hold no |
| 244 | communicator; they only iterate the source to stay in the trainer-side |
| 245 | collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the |
| 246 | broadcast (all guarded on `is_sender`). |
| 247 | |
| 248 | `packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender |
| 249 | propagates them to the worker at `trainer_init` (on the worker-side init |
| 250 | info), so per-round payloads carry only parameter metadata. |
| 251 | """ |
| 252 | |
| 253 | init_info_cls = NCCLTrainerInitInfo |
| 254 | |
| 255 | def __init__( |
| 256 | self, |
| 257 | *, |
| 258 | client: VLLMWeightSyncClient, |
| 259 | source: WeightSource, |
| 260 | is_sender: bool = True, |
| 261 | packed: bool = True, |
| 262 | packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, |
| 263 | packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS, |
| 264 | ) -> None: |
| 265 | super().__init__(client=client, source=source, is_sender=is_sender) |
| 266 | self.packed = packed |
| 267 | self.packed_buffer_size_bytes = packed_buffer_size_bytes |
| 268 | self.packed_num_buffers = packed_num_buffers |
| 269 | self.model_update_group: PyNcclCommunicator | None = None |
| 270 | |
| 271 | @classmethod |
| 272 | def trainer_init( |
| 273 | cls, |
| 274 | init_info: NCCLTrainerInitInfo, |
| 275 | *, |
| 276 | client: VLLMWeightSyncClient, |
| 277 | source: WeightSource | None = None, |
| 278 | ) -> Self: |
| 279 | if source is None: |
| 280 | raise ValueError("NCCL trainer weight transfer requires a WeightSource.") |
| 281 | engine = cls( |
| 282 | client=client, |
| 283 | source=source, |
| 284 | is_sender=init_info.is_sender, |
| 285 | packed=init_info.packed, |
| 286 | packed_buffer_size_bytes=init_info.packed_buffer_size_bytes, |
| 287 | packed_num_buffers=init_info.packed_num_buffers, |
| 288 | ) |
| 289 | if not engine.is_sender: |
| 290 | # Non-sender trainer ranks aren't part of the transfer NCCL group and |
| 291 | # don't drive the inference side; they only participate in the |
| 292 | # trainer-side gather during send_weights. |
| 293 | return engine |
| 294 | |
| 295 | # Workers sit at rank_offset 1, after the single trainer sender rank 0. |
| 296 | worker_init_info = NCCLWeightTransferInitInfo( |
| 297 | master_address=init_info.master_address, |
| 298 | master_port=init_info.master_port, |
| 299 | rank_offset=1, |
| 300 | world_size=init_info.world_size, |
| 301 | packed=init_info.packed, |
| 302 | packed_buffer_size_bytes=init_info.packed_buffer_size_bytes, |
| 303 | packed_num_buffers=init_info.packed_num_buffers, |
| 304 | ) |
| 305 | |
| 306 | # The inference workers block inside init_weight_transfer_engine waiting |
| 307 | # for the NCCL rendezvous, so we kick that off on a side thread while we |
| 308 | # open the trainer endpoint (rank 0); both sides must rendezvous together. |
| 309 | with ThreadPoolExecutor(max_workers=1) as exe: |
| 310 | future = exe.submit( |
| 311 | engine.client.init_weight_transfer_engine, |
| 312 | worker_init_payload(worker_init_info), |
| 313 | ) |
| 314 | engine.model_update_group = open_trainer_endpoint(init_info) |
| 315 | future.result() # surface any inference-side init error |
| 316 | |
| 317 | return engine |
| 318 | |
| 319 | def send_weights(self) -> None: |
| 320 | assert self.source is not None # guaranteed by trainer_init / __init__ |
| 321 | source = self.source |
| 322 | |
| 323 | # Metadata is declared without gathering. For Megatron it is itself a |
| 324 | # collective, so every rank runs it; only the sender ships it. |
| 325 | meta = source.metadata() |
| 326 | |
| 327 | if not self.is_sender: |
| 328 | # Non-sender ranks only join the trainer-side gather collective. |
| 329 | self._broadcast(source, meta) |
| 330 | self._post_send_sync() |
| 331 | return |
| 332 | |
| 333 | update_info = NCCLWeightTransferUpdateInfo( |
| 334 | names=[m.name for m in meta], |
| 335 | dtype_names=[str(m.dtype).split(".")[-1] for m in meta], |
| 336 | shapes=[list(m.shape) for m in meta], |
| 337 | ) |
| 338 | |
| 339 | self.client.start_weight_update() |
| 340 | # update_weights (workers receive) must run concurrently with the |
| 341 | # trainer-side broadcast — both rendezvous inside the same NCCL calls. |
| 342 | exe = ThreadPoolExecutor(max_workers=1) |
| 343 | try: |
| 344 | future = exe.submit(self.client.update_weights, asdict(update_info)) |
| 345 | # Cheap best-effort: if update_weights already failed (e.g. a bad |
| 346 | # request rejected before any NCCL call), surface it now instead of |
| 347 | # hanging in broadcast waiting for a peer that will never arrive. |
| 348 | if future.done(): |
| 349 | future.result() |
| 350 | self._broadcast(source, meta) |
| 351 | future.result() # surface inference-side errors |
| 352 | finally: |
| 353 | # Never wait for the RPC thread here. If the broadcast raised, the |
| 354 | # worker is still blocked in the matching NCCL call and will never |
| 355 | # return, so joining would turn the error into a permanent hang. |
| 356 | # Let the exception out instead; the transfer group is unusable |
| 357 | # either way and the caller has to tear it down. (The orphaned |
| 358 | # thread is still joined at interpreter exit, so a caller that wants |
| 359 | # to exit cleanly after such a failure must not wait on it.) |
| 360 | exe.shutdown(wait=False) |
| 361 | self.client.finish_weight_update() |
| 362 | self._post_send_sync() |
| 363 | |
| 364 | def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None: |
| 365 | """Iterate the source (materializing each tensor — a collective on all |
| 366 | ranks) and, on the sender, broadcast from rank 0, packed or one-by-one. |
| 367 | Non-sender ranks only replay the iteration to stay in the collective.""" |
| 368 | if not self.is_sender: |
| 369 | for _ in source: |
| 370 | pass |
| 371 | return |
| 372 | |
| 373 | assert self.model_update_group is not None, ( |
| 374 | "trainer_init() must be called before _broadcast()." |
| 375 | ) |
| 376 | pairs = self._checked_iter(source, meta) |
| 377 | if self.packed: |
| 378 | packed_nccl_broadcast_producer( |
| 379 | iterator=pairs, |
| 380 | group=self.model_update_group, |
| 381 | src=0, |
| 382 | post_iter_func=lambda item: item[1], |
| 383 | buffer_size_bytes=self.packed_buffer_size_bytes, |
| 384 | num_buffers=self.packed_num_buffers, |
| 385 | ) |
| 386 | else: |
| 387 | stream = torch.cuda.current_stream() |
| 388 | for _name, tensor in pairs: |
| 389 | # NCCL sends `numel` elements straight from `data_ptr()`, so a |
| 390 | # non-contiguous view would ship whatever follows its base |
| 391 | # pointer. Keep the copy referenced until the broadcast is |
| 392 | # enqueued. (The packed path linearizes in `pack_tensors`.) |
| 393 | send = tensor if tensor.is_contiguous() else tensor.contiguous() |
| 394 | self.model_update_group.broadcast(send, src=0, stream=stream) |
| 395 | |
| 396 | @staticmethod |
| 397 | def _checked_iter( |
| 398 | source: WeightSource, meta: list[ParamMeta] |
| 399 | ) -> Iterator[tuple[str, torch.Tensor]]: |
| 400 | """Yield the source's pairs, checking each against what the worker was |
| 401 | told to expect. |
| 402 | |
| 403 | The worker sizes its receive buffers — and in packed mode cuts its chunk |
| 404 | boundaries — from the update info, which is built from `metadata()`. If |
| 405 | iteration disagrees with it, the two sides split the stream differently |
| 406 | and the transfer hangs in NCCL or loads garbage. Checking here costs one |
| 407 | comparison per parameter and turns that into an error naming the first |
| 408 | divergent parameter. Sender-only: under pipeline parallelism a |
| 409 | non-sender's yielded tensor is not meaningful. |
| 410 | """ |
| 411 | sent = 0 |
| 412 | for name, tensor in source: |
| 413 | if sent >= len(meta): |
| 414 | raise ValueError( |
| 415 | f"WeightSource yielded more parameters than metadata() " |
| 416 | f"declared ({len(meta)}); first extra is {name!r}." |
| 417 | ) |
| 418 | expected = meta[sent] |
| 419 | if ( |
| 420 | name != expected.name |
| 421 | or tensor.dtype != expected.dtype |
| 422 | or tuple(tensor.shape) != expected.shape |
| 423 | ): |
| 424 | raise ValueError( |
| 425 | "WeightSource metadata() disagrees with iteration at index " |
| 426 | f"{sent}: declared {expected.name!r} " |
| 427 | f"{expected.dtype} {tuple(expected.shape)}, got {name!r} " |
| 428 | f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must " |
| 429 | "enumerate the same parameters in the same order." |
| 430 | ) |
| 431 | sent += 1 |
| 432 | yield name, tensor |
| 433 | if sent != len(meta): |
| 434 | raise ValueError( |
| 435 | f"WeightSource yielded {sent} parameters but metadata() " |
| 436 | f"declared {len(meta)}; the worker is waiting for the rest." |
| 437 | ) |
| 438 | |
| 439 | def _post_send_sync(self) -> None: |
| 440 | """Wait for this rank's transfer work to land before returning. |
| 441 | |
| 442 | Broadcasts are only *enqueued* by `send_weights`: the unpacked path on |
| 443 | the current stream, the packed path on the producer's own streams (which |
| 444 | it drains itself). Waiting here lets a caller mutate parameters, or |
| 445 | start the next step on another stream, as soon as `send_weights` |
| 446 | returns, instead of silently depending on same-stream ordering. Every |
| 447 | rank waits: a non-sender's `full_tensor()` gathers feed the sender's |
| 448 | broadcast, so they must have landed before it may touch its shards. |
| 449 | |
| 450 | Unlike IPC there is no cross-rank barrier here. Nothing in this backend |
| 451 | outlives the collective it travelled in (IPC's barrier keeps shared |
| 452 | buffers alive until every consumer has opened them), so a barrier would |
| 453 | only add a dependency on the default process group that this backend |
| 454 | otherwise does not have. |
| 455 | """ |
| 456 | if torch.cuda.is_available(): |
| 457 | torch.cuda.current_stream().synchronize() |
| 458 | |
| 459 | def shutdown(self) -> None: |
| 460 | self.model_update_group = None |