@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d4d90536f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Distributed] Add send and recv helpers (#5719)

Murali Andoorveedu committed Jun 23, 2024 at 17:42 UTC 5d4d90536fa24c032bb91ae629b7b4958e045b03
6 files changed +278 -24
tests/distributed/test_comm_ops.py
+74 -4
@@ -8,12 +8,11 @@ import pytest
8 import ray
9 import torch
10
11 -from vllm.distributed import (broadcast_tensor_dict,
11 +from vllm.distributed import (broadcast_tensor_dict, get_pp_group,
12 tensor_model_parallel_all_gather,
13 tensor_model_parallel_all_reduce)
14
15 -from ..utils import (init_test_distributed_environment,
16 - multi_process_tensor_parallel)
15 +from ..utils import init_test_distributed_environment, multi_process_parallel
16
17
18 @ray.remote(num_gpus=1, max_calls=1)
@@ -105,6 +104,68 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
104 assert torch.allclose(recv_dict["f"], test_dict["f"])
105
106
107 +@ray.remote(num_gpus=1, max_calls=1)
108 +def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
109 + distributed_init_port: str):
110 + del os.environ["CUDA_VISIBLE_DEVICES"]
111 + device = torch.device(f"cuda:{rank}")
112 + torch.cuda.set_device(device)
113 + init_test_distributed_environment(tp_size, pp_size, rank,
114 + distributed_init_port)
115 +
116 + test_dict = {
117 + # device tensor
118 + "a": torch.arange(8, dtype=torch.float32, device="cuda"),
119 + # CPU tensor
120 + "b": torch.arange(16, dtype=torch.int8, device="cpu"),
121 + "c": "test",
122 + "d": [1, 2, 3],
123 + "e": {
124 + "a": 1,
125 + "b": 2
126 + },
127 + # empty tensor
128 + "f": torch.tensor([], dtype=torch.float32, device="cuda"),
129 + }
130 +
131 + if not get_pp_group().is_first_rank:
132 + recv_dict = get_pp_group().recv_tensor_dict()
133 +
134 + if not get_pp_group().is_last_rank:
135 + get_pp_group().send_tensor_dict(test_dict)
136 +
137 + if not get_pp_group().is_first_rank:
138 + assert len(recv_dict) == len(test_dict)
139 + assert torch.allclose(recv_dict["a"], test_dict["a"])
140 + assert torch.allclose(recv_dict["b"], test_dict["b"])
141 + assert recv_dict["c"] == test_dict["c"]
142 + assert recv_dict["d"] == test_dict["d"]
143 + assert recv_dict["e"] == test_dict["e"]
144 + assert torch.allclose(recv_dict["f"], test_dict["f"])
145 +
146 +
147 +@ray.remote(num_gpus=1, max_calls=1)
148 +def send_recv_test_worker(tp_size: int, pp_size: int, rank: int,
149 + distributed_init_port: str):
150 + del os.environ["CUDA_VISIBLE_DEVICES"]
151 + device = torch.device(f"cuda:{rank}")
152 + torch.cuda.set_device(device)
153 + init_test_distributed_environment(tp_size, pp_size, rank,
154 + distributed_init_port)
155 +
156 + size = 64
157 + test_tensor = torch.arange(64, dtype=torch.float32, device="cuda")
158 +
159 + if not get_pp_group().is_first_rank:
160 + recv_tensor = get_pp_group().recv(size, dtype=torch.float32)
161 +
162 + if not get_pp_group().is_last_rank:
163 + get_pp_group().send(test_tensor)
164 +
165 + if not get_pp_group().is_first_rank:
166 + assert torch.allclose(test_tensor, recv_tensor)
167 +
168 +
169 @pytest.mark.skipif(torch.cuda.device_count() < 2,
170 reason="Need at least 2 GPUs to run the test.")
171 @pytest.mark.parametrize("tp_size", [2])
@@ -113,4 +174,13 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
174 broadcast_tensor_dict_test_worker
175 ])
176 def test_multi_process_tensor_parallel(tp_size, test_target):
116 - multi_process_tensor_parallel(tp_size, 1, test_target)
177 + multi_process_parallel(tp_size, 1, test_target)
178 +
179 +
180 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
181 + reason="Need at least 2 GPUs to run the test.")
182 +@pytest.mark.parametrize("pp_size", [2])
183 +@pytest.mark.parametrize(
184 + "test_target", [send_recv_test_worker, send_recv_tensor_dict_test_worker])
185 +def test_multi_process_pipeline_parallel(pp_size, test_target):
186 + multi_process_parallel(1, pp_size, test_target)
tests/distributed/test_custom_all_reduce.py
+2 -3
@@ -12,8 +12,7 @@ from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12 get_tp_group, graph_capture)
13
14 from ..utils import (ensure_model_parallel_initialized,
15 - init_test_distributed_environment,
16 - multi_process_tensor_parallel)
15 + init_test_distributed_environment, multi_process_parallel)
16
17 random.seed(42)
18 test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
@@ -113,4 +112,4 @@ def test_custom_allreduce(tp_size, pipeline_parallel_size, test_target):
112 world_size = tp_size * pipeline_parallel_size
113 if world_size > torch.cuda.device_count():
114 pytest.skip("Not enough GPUs to run the test.")
116 - multi_process_tensor_parallel(tp_size, pipeline_parallel_size, test_target)
115 + multi_process_parallel(tp_size, pipeline_parallel_size, test_target)
tests/distributed/test_pynccl.py
+12 -4
@@ -168,9 +168,13 @@ def send_recv_worker_fn():
168 dtype=torch.float32).cuda(pynccl_comm.rank)
169 with pynccl_comm.change_state(enable=True):
170 if pynccl_comm.rank == 0:
171 - pynccl_comm.send(tensor)
171 + pynccl_comm.send(tensor,
172 + dst=(pynccl_comm.rank + 1) %
173 + pynccl_comm.world_size)
174 else:
173 - pynccl_comm.recv(tensor)
175 + pynccl_comm.recv(tensor,
176 + src=(pynccl_comm.rank - 1) %
177 + pynccl_comm.world_size)
178 result = tensor.mean().cpu().item()
179 assert result == 1
180
@@ -203,9 +207,13 @@ def multiple_send_recv_worker_fn():
207 device=device)
208 with pynccl_comm.change_state(enable=True):
209 if torch.distributed.get_rank() in [0, 1]:
206 - pynccl_comm.send(tensor)
210 + pynccl_comm.send(tensor,
211 + dst=(pynccl_comm.rank + 1) %
212 + pynccl_comm.world_size)
213 else:
208 - pynccl_comm.recv(tensor)
214 + pynccl_comm.recv(tensor,
215 + src=(pynccl_comm.rank - 1) %
216 + pynccl_comm.world_size)
217 result = tensor.mean().cpu().item()
218 if torch.distributed.get_rank() in [0, 2]:
219 assert result == 1
tests/utils.py
+1 -1
@@ -129,7 +129,7 @@ def init_test_distributed_environment(
129 ensure_model_parallel_initialized(tp_size, pp_size)
130
131
132 -def multi_process_tensor_parallel(
132 +def multi_process_parallel(
133 tp_size: int,
134 pp_size: int,
135 test_target,
vllm/distributed/device_communicators/pynccl.py
+2 -12
@@ -121,10 +121,7 @@ class PyNcclCommunicator:
121 ncclRedOpTypeEnum.from_torch(op), self.comm,
122 cudaStream_t(stream.cuda_stream))
123
124 - def send(self,
125 - tensor: torch.Tensor,
126 - dst: Optional[int] = None,
127 - stream=None):
124 + def send(self, tensor: torch.Tensor, dst: int, stream=None):
125 if self.disabled:
126 return
127 assert tensor.device == self.device, (
@@ -132,16 +129,11 @@ class PyNcclCommunicator:
129 f"but the input tensor is on {tensor.device}")
130 if stream is None:
131 stream = self.stream
135 - if dst is None:
136 - dst = (self.rank + 1) % self.world_size
132 self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
133 ncclDataTypeEnum.from_torch(tensor.dtype), dst,
134 self.comm, cudaStream_t(stream.cuda_stream))
135
141 - def recv(self,
142 - tensor: torch.Tensor,
143 - src: Optional[int] = None,
144 - stream=None):
136 + def recv(self, tensor: torch.Tensor, src: int, stream=None):
137 if self.disabled:
138 return
139 assert tensor.device == self.device, (
@@ -149,8 +141,6 @@ class PyNcclCommunicator:
141 f"but the input tensor is on {tensor.device}")
142 if stream is None:
143 stream = self.stream
152 - if src is None:
153 - src = (self.rank - 1) % self.world_size
144 self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
145 ncclDataTypeEnum.from_torch(tensor.dtype), src,
146 self.comm, cudaStream_t(stream.cuda_stream))
vllm/distributed/parallel_state.py
+187
@@ -20,6 +20,7 @@ If you only need to use the distributed environment without model/pipeline
20 steps.
21 """
22 import contextlib
23 +import pickle
24 from collections import namedtuple
25 from contextlib import contextmanager, nullcontext
26 from dataclasses import dataclass
@@ -28,6 +29,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
29 from unittest.mock import patch
30
31 import torch
32 +import torch.distributed
33 from torch.distributed import Backend, ProcessGroup
34
35 import vllm.envs as envs
@@ -180,6 +182,16 @@ class GroupCoordinator:
182 """Return the global rank of the last process in the group"""
183 return self.ranks[-1]
184
185 + @property
186 + def is_first_rank(self):
187 + """Return whether the caller is the first process in the group"""
188 + return self.rank == self.first_rank
189 +
190 + @property
191 + def is_last_rank(self):
192 + """Return whether the caller is the last process in the group"""
193 + return self.rank == self.last_rank
194 +
195 @property
196 def next_rank(self):
197 """Return the global rank of the process that follows the caller"""
@@ -374,6 +386,70 @@ class GroupCoordinator:
386 group=self.device_group)
387 return obj_list
388
389 + def send_object(self, obj: Any, dst: int) -> None:
390 + """Send the input object list to the destination rank."""
391 + """NOTE: `dst` is the local rank of the destination rank."""
392 +
393 + assert dst < self.world_size, f"Invalid dst rank ({dst})"
394 +
395 + assert dst != self.rank, (
396 + "Invalid destination rank. Destination rank is the same "
397 + "as the current rank.")
398 +
399 + # Serialize object to tensor and get the size as well
400 + object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8)
401 +
402 + size_tensor = torch.tensor([object_tensor.numel()],
403 + dtype=torch.long,
404 + device="cpu")
405 +
406 + # Send object size
407 +
408 + torch.distributed.send(size_tensor,
409 + dst=self.ranks[dst],
410 + group=self.cpu_group)
411 +
412 + # Send object
413 + torch.distributed.send(object_tensor,
414 + dst=self.ranks[dst],
415 + group=self.cpu_group)
416 +
417 + return None
418 +
419 + def recv_object(self, src: int) -> Any:
420 + """Receive the input object list from the source rank."""
421 + """NOTE: `src` is the local rank of the source rank."""
422 +
423 + assert src < self.world_size, f"Invalid src rank ({src})"
424 +
425 + assert src != self.rank, (
426 + "Invalid source rank. Source rank is the same as the current rank."
427 + )
428 +
429 + size_tensor = torch.empty(1, dtype=torch.long, device="cpu")
430 +
431 + # Receive object size
432 + rank_size = torch.distributed.recv(size_tensor,
433 + src=src,
434 + group=self.cpu_group)
435 +
436 + # Tensor to receive serialized objects into.
437 + object_tensor = torch.empty( # type: ignore[call-overload]
438 + size_tensor.item(), # type: ignore[arg-type]
439 + dtype=torch.uint8,
440 + device="cpu")
441 +
442 + rank_object = torch.distributed.recv(object_tensor,
443 + src=src,
444 + group=self.cpu_group)
445 +
446 + assert rank_object == rank_size, (
447 + "Received object sender rank does not match the size sender rank.")
448 +
449 + obj = pickle.loads(object_tensor.numpy().tobytes())
450 +
451 + return obj
452 +
453 def broadcast_tensor_dict(
454 self,
455 tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
@@ -459,6 +535,88 @@ class GroupCoordinator:
535 async_handle.wait()
536 return tensor_dict
537
538 + def send_tensor_dict(
539 + self,
540 + tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
541 + dst: Optional[int] = None
542 + ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
543 + """Send the input tensor dictionary.
544 + NOTE: `dst` is the local rank of the source rank.
545 + """
546 + # Bypass the function if we are using only 1 GPU.
547 + if not torch.distributed.is_initialized() or self.world_size == 1:
548 + return tensor_dict
549 +
550 + group = self.device_group
551 + metadata_group = self.cpu_group
552 +
553 + if dst is None:
554 + dst = self.next_rank
555 + assert dst < self.world_size, f"Invalid dst rank ({dst})"
556 +
557 + metadata_list: List[Tuple[Any, Any]] = []
558 + assert isinstance(
559 + tensor_dict,
560 + dict), f"Expecting a dictionary, got {type(tensor_dict)}"
561 + metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
562 + # `metadata_list` lives in CPU memory.
563 + # `send_object_list` has serialization & deserialization,
564 + # all happening on CPU. Therefore, we can use the CPU group.
565 + self.send_object(metadata_list, dst=dst)
566 + for tensor in tensor_list:
567 + if tensor.numel() == 0:
568 + # Skip sending empty tensors.
569 + continue
570 + if tensor.is_cpu:
571 + # use metadata_group for CPU tensors
572 + torch.distributed.send(tensor, dst=dst, group=metadata_group)
573 + else:
574 + # use group for GPU tensors
575 + torch.distributed.send(tensor, dst=dst, group=group)
576 + return None
577 +
578 + def recv_tensor_dict(
579 + self,
580 + src: Optional[int] = None
581 + ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
582 + """Recv the input tensor dictionary.
583 + NOTE: `src` is the local rank of the source rank.
584 + """
585 + # Bypass the function if we are using only 1 GPU.
586 + if not torch.distributed.is_initialized() or self.world_size == 1:
587 + return None
588 +
589 + group = self.device_group
590 + metadata_group = self.cpu_group
591 +
592 + if src is None:
593 + src = self.prev_rank
594 + assert src < self.world_size, f"Invalid src rank ({src})"
595 +
596 + recv_metadata_list = self.recv_object(src=src)
597 + tensor_dict = {}
598 + for key, value in recv_metadata_list:
599 + if isinstance(value, TensorMetadata):
600 + tensor = torch.empty(value.size,
601 + dtype=value.dtype,
602 + device=value.device)
603 + if tensor.numel() == 0:
604 + # Skip broadcasting empty tensors.
605 + tensor_dict[key] = tensor
606 + continue
607 + if tensor.is_cpu:
608 + # use metadata_group for CPU tensors
609 + torch.distributed.recv(tensor,
610 + src=src,
611 + group=metadata_group)
612 + else:
613 + # use group for GPU tensors
614 + torch.distributed.recv(tensor, src=src, group=group)
615 + tensor_dict[key] = tensor
616 + else:
617 + tensor_dict[key] = value
618 + return tensor_dict
619 +
620 def barrier(self):
621 """Barrier synchronization among the group.
622 NOTE: don't use `device_group` here! `barrier` in NCCL is
@@ -468,6 +626,35 @@ class GroupCoordinator:
626 """
627 torch.distributed.barrier(group=self.cpu_group)
628
629 + def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
630 + """Sends a tensor to the destination rank in a non-blocking way"""
631 + """NOTE: `dst` is the local rank of the destination rank."""
632 + if dst is None:
633 + dst = self.next_rank
634 +
635 + pynccl_comm = self.pynccl_comm
636 + if pynccl_comm is not None and not pynccl_comm.disabled:
637 + pynccl_comm.send(tensor, dst)
638 + else:
639 + torch.distributed.send(tensor, self.ranks[dst], self.device_group)
640 +
641 + def recv(self,
642 + size: torch.Size,
643 + dtype: torch.dtype,
644 + src: Optional[int] = None) -> torch.Tensor:
645 + """Receives a tensor from the src rank."""
646 + """NOTE: `src` is the local rank of the destination rank."""
647 + if src is None:
648 + src = self.prev_rank
649 +
650 + tensor = torch.empty(size, dtype=dtype, device=self.device)
651 + pynccl_comm = self.pynccl_comm
652 + if pynccl_comm is not None and not pynccl_comm.disabled:
653 + pynccl_comm.recv(tensor, src)
654 + else:
655 + torch.distributed.recv(tensor, self.ranks[src], self.device_group)
656 + return tensor
657 +
658 def destroy(self):
659 if self.device_group is not None:
660 torch.distributed.destroy_process_group(self.device_group)