A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Distributed] Add send and recv helpers (#5719)
Murali Andoorveedu committed
Jun 23, 2024 at 17:42 UTC
5d4d90536fa24c032bb91ae629b7b4958e045b03
6 files changed
+278
-24
tests/distributed/test_comm_ops.py
+74
-4
@@ -8,12 +8,11 @@ import pytest
8
import ray
9
import torch
10
11
-from vllm.distributed import (broadcast_tensor_dict,
11
+from vllm.distributed import (broadcast_tensor_dict, get_pp_group,
12
tensor_model_parallel_all_gather,
13
tensor_model_parallel_all_reduce)
14
15
-from ..utils import (init_test_distributed_environment,
16
- multi_process_tensor_parallel)
15
+from ..utils import init_test_distributed_environment, multi_process_parallel
16
17
18
@ray.remote(num_gpus=1, max_calls=1)
@@ -105,6 +104,68 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
104
assert torch.allclose(recv_dict["f"], test_dict["f"])
105
106
107
+@ray.remote(num_gpus=1, max_calls=1)
108
+def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
109
+ distributed_init_port: str):
110
+ del os.environ["CUDA_VISIBLE_DEVICES"]
111
+ device = torch.device(f"cuda:{rank}")
112
+ torch.cuda.set_device(device)
113
+ init_test_distributed_environment(tp_size, pp_size, rank,
114
+ distributed_init_port)
115
+
116
+ test_dict = {
117
+ # device tensor
118
+ "a": torch.arange(8, dtype=torch.float32, device="cuda"),
119
+ # CPU tensor
120
+ "b": torch.arange(16, dtype=torch.int8, device="cpu"),
121
+ "c": "test",
122
+ "d": [1, 2, 3],
123
+ "e": {
124
+ "a": 1,
125
+ "b": 2
126
+ },
127
+ # empty tensor
128
+ "f": torch.tensor([], dtype=torch.float32, device="cuda"),
129
+ }
130
+
131
+ if not get_pp_group().is_first_rank:
132
+ recv_dict = get_pp_group().recv_tensor_dict()
133
+
134
+ if not get_pp_group().is_last_rank:
135
+ get_pp_group().send_tensor_dict(test_dict)
136
+
137
+ if not get_pp_group().is_first_rank:
138
+ assert len(recv_dict) == len(test_dict)
139
+ assert torch.allclose(recv_dict["a"], test_dict["a"])
140
+ assert torch.allclose(recv_dict["b"], test_dict["b"])
141
+ assert recv_dict["c"] == test_dict["c"]
142
+ assert recv_dict["d"] == test_dict["d"]
143
+ assert recv_dict["e"] == test_dict["e"]
144
+ assert torch.allclose(recv_dict["f"], test_dict["f"])
145
+
146
+
147
+@ray.remote(num_gpus=1, max_calls=1)
148
+def send_recv_test_worker(tp_size: int, pp_size: int, rank: int,
149
+ distributed_init_port: str):
150
+ del os.environ["CUDA_VISIBLE_DEVICES"]
151
+ device = torch.device(f"cuda:{rank}")
152
+ torch.cuda.set_device(device)
153
+ init_test_distributed_environment(tp_size, pp_size, rank,
154
+ distributed_init_port)
155
+
156
+ size = 64
157
+ test_tensor = torch.arange(64, dtype=torch.float32, device="cuda")
158
+
159
+ if not get_pp_group().is_first_rank:
160
+ recv_tensor = get_pp_group().recv(size, dtype=torch.float32)
161
+
162
+ if not get_pp_group().is_last_rank:
163
+ get_pp_group().send(test_tensor)
164
+
165
+ if not get_pp_group().is_first_rank:
166
+ assert torch.allclose(test_tensor, recv_tensor)
167
+
168
+
169
@pytest.mark.skipif(torch.cuda.device_count() < 2,
170
reason="Need at least 2 GPUs to run the test.")
171
@pytest.mark.parametrize("tp_size", [2])
@@ -113,4 +174,13 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
174
broadcast_tensor_dict_test_worker
175
])
176
def test_multi_process_tensor_parallel(tp_size, test_target):
116
- multi_process_tensor_parallel(tp_size, 1, test_target)
177
+ multi_process_parallel(tp_size, 1, test_target)
178
+
179
+
180
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
181
+ reason="Need at least 2 GPUs to run the test.")
182
+@pytest.mark.parametrize("pp_size", [2])
183
+@pytest.mark.parametrize(
184
+ "test_target", [send_recv_test_worker, send_recv_tensor_dict_test_worker])
185
+def test_multi_process_pipeline_parallel(pp_size, test_target):
186
+ multi_process_parallel(1, pp_size, test_target)
tests/distributed/test_custom_all_reduce.py
+2
-3
@@ -12,8 +12,7 @@ from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12
get_tp_group, graph_capture)
13
14
from ..utils import (ensure_model_parallel_initialized,
15
- init_test_distributed_environment,
16
- multi_process_tensor_parallel)
15
+ init_test_distributed_environment, multi_process_parallel)
16
17
random.seed(42)
18
test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
@@ -113,4 +112,4 @@ def test_custom_allreduce(tp_size, pipeline_parallel_size, test_target):
112
world_size = tp_size * pipeline_parallel_size
113
if world_size > torch.cuda.device_count():
114
pytest.skip("Not enough GPUs to run the test.")
116
- multi_process_tensor_parallel(tp_size, pipeline_parallel_size, test_target)
115
+ multi_process_parallel(tp_size, pipeline_parallel_size, test_target)
tests/distributed/test_pynccl.py
+12
-4
@@ -168,9 +168,13 @@ def send_recv_worker_fn():
168
dtype=torch.float32).cuda(pynccl_comm.rank)
169
with pynccl_comm.change_state(enable=True):
170
if pynccl_comm.rank == 0:
171
- pynccl_comm.send(tensor)
171
+ pynccl_comm.send(tensor,
172
+ dst=(pynccl_comm.rank + 1) %
173
+ pynccl_comm.world_size)
174
else:
173
- pynccl_comm.recv(tensor)
175
+ pynccl_comm.recv(tensor,
176
+ src=(pynccl_comm.rank - 1) %
177
+ pynccl_comm.world_size)
178
result = tensor.mean().cpu().item()
179
assert result == 1
180
@@ -203,9 +207,13 @@ def multiple_send_recv_worker_fn():
207
device=device)
208
with pynccl_comm.change_state(enable=True):
209
if torch.distributed.get_rank() in [0, 1]:
206
- pynccl_comm.send(tensor)
210
+ pynccl_comm.send(tensor,
211
+ dst=(pynccl_comm.rank + 1) %
212
+ pynccl_comm.world_size)
213
else:
208
- pynccl_comm.recv(tensor)
214
+ pynccl_comm.recv(tensor,
215
+ src=(pynccl_comm.rank - 1) %
216
+ pynccl_comm.world_size)
217
result = tensor.mean().cpu().item()
218
if torch.distributed.get_rank() in [0, 2]:
219
assert result == 1
tests/utils.py
+1
-1
@@ -129,7 +129,7 @@ def init_test_distributed_environment(
129
ensure_model_parallel_initialized(tp_size, pp_size)
130
131
132
-def multi_process_tensor_parallel(
132
+def multi_process_parallel(
133
tp_size: int,
134
pp_size: int,
135
test_target,
vllm/distributed/device_communicators/pynccl.py
+2
-12
@@ -121,10 +121,7 @@ class PyNcclCommunicator:
121
ncclRedOpTypeEnum.from_torch(op), self.comm,
122
cudaStream_t(stream.cuda_stream))
123
124
- def send(self,
125
- tensor: torch.Tensor,
126
- dst: Optional[int] = None,
127
- stream=None):
124
+ def send(self, tensor: torch.Tensor, dst: int, stream=None):
125
if self.disabled:
126
return
127
assert tensor.device == self.device, (
@@ -132,16 +129,11 @@ class PyNcclCommunicator:
129
f"but the input tensor is on {tensor.device}")
130
if stream is None:
131
stream = self.stream
135
- if dst is None:
136
- dst = (self.rank + 1) % self.world_size
132
self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
133
ncclDataTypeEnum.from_torch(tensor.dtype), dst,
134
self.comm, cudaStream_t(stream.cuda_stream))
135
141
- def recv(self,
142
- tensor: torch.Tensor,
143
- src: Optional[int] = None,
144
- stream=None):
136
+ def recv(self, tensor: torch.Tensor, src: int, stream=None):
137
if self.disabled:
138
return
139
assert tensor.device == self.device, (
@@ -149,8 +141,6 @@ class PyNcclCommunicator:
141
f"but the input tensor is on {tensor.device}")
142
if stream is None:
143
stream = self.stream
152
- if src is None:
153
- src = (self.rank - 1) % self.world_size
144
self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
145
ncclDataTypeEnum.from_torch(tensor.dtype), src,
146
self.comm, cudaStream_t(stream.cuda_stream))
vllm/distributed/parallel_state.py
+187
@@ -20,6 +20,7 @@ If you only need to use the distributed environment without model/pipeline
20
steps.
21
"""
22
import contextlib
23
+import pickle
24
from collections import namedtuple
25
from contextlib import contextmanager, nullcontext
26
from dataclasses import dataclass
@@ -28,6 +29,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
29
from unittest.mock import patch
30
31
import torch
32
+import torch.distributed
33
from torch.distributed import Backend, ProcessGroup
34
35
import vllm.envs as envs
@@ -180,6 +182,16 @@ class GroupCoordinator:
182
"""Return the global rank of the last process in the group"""
183
return self.ranks[-1]
184
185
+ @property
186
+ def is_first_rank(self):
187
+ """Return whether the caller is the first process in the group"""
188
+ return self.rank == self.first_rank
189
+
190
+ @property
191
+ def is_last_rank(self):
192
+ """Return whether the caller is the last process in the group"""
193
+ return self.rank == self.last_rank
194
+
195
@property
196
def next_rank(self):
197
"""Return the global rank of the process that follows the caller"""
@@ -374,6 +386,70 @@ class GroupCoordinator:
386
group=self.device_group)
387
return obj_list
388
389
+ def send_object(self, obj: Any, dst: int) -> None:
390
+ """Send the input object list to the destination rank."""
391
+ """NOTE: `dst` is the local rank of the destination rank."""
392
+
393
+ assert dst < self.world_size, f"Invalid dst rank ({dst})"
394
+
395
+ assert dst != self.rank, (
396
+ "Invalid destination rank. Destination rank is the same "
397
+ "as the current rank.")
398
+
399
+ # Serialize object to tensor and get the size as well
400
+ object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8)
401
+
402
+ size_tensor = torch.tensor([object_tensor.numel()],
403
+ dtype=torch.long,
404
+ device="cpu")
405
+
406
+ # Send object size
407
+
408
+ torch.distributed.send(size_tensor,
409
+ dst=self.ranks[dst],
410
+ group=self.cpu_group)
411
+
412
+ # Send object
413
+ torch.distributed.send(object_tensor,
414
+ dst=self.ranks[dst],
415
+ group=self.cpu_group)
416
+
417
+ return None
418
+
419
+ def recv_object(self, src: int) -> Any:
420
+ """Receive the input object list from the source rank."""
421
+ """NOTE: `src` is the local rank of the source rank."""
422
+
423
+ assert src < self.world_size, f"Invalid src rank ({src})"
424
+
425
+ assert src != self.rank, (
426
+ "Invalid source rank. Source rank is the same as the current rank."
427
+ )
428
+
429
+ size_tensor = torch.empty(1, dtype=torch.long, device="cpu")
430
+
431
+ # Receive object size
432
+ rank_size = torch.distributed.recv(size_tensor,
433
+ src=src,
434
+ group=self.cpu_group)
435
+
436
+ # Tensor to receive serialized objects into.
437
+ object_tensor = torch.empty( # type: ignore[call-overload]
438
+ size_tensor.item(), # type: ignore[arg-type]
439
+ dtype=torch.uint8,
440
+ device="cpu")
441
+
442
+ rank_object = torch.distributed.recv(object_tensor,
443
+ src=src,
444
+ group=self.cpu_group)
445
+
446
+ assert rank_object == rank_size, (
447
+ "Received object sender rank does not match the size sender rank.")
448
+
449
+ obj = pickle.loads(object_tensor.numpy().tobytes())
450
+
451
+ return obj
452
+
453
def broadcast_tensor_dict(
454
self,
455
tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
@@ -459,6 +535,88 @@ class GroupCoordinator:
535
async_handle.wait()
536
return tensor_dict
537
538
+ def send_tensor_dict(
539
+ self,
540
+ tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
541
+ dst: Optional[int] = None
542
+ ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
543
+ """Send the input tensor dictionary.
544
+ NOTE: `dst` is the local rank of the source rank.
545
+ """
546
+ # Bypass the function if we are using only 1 GPU.
547
+ if not torch.distributed.is_initialized() or self.world_size == 1:
548
+ return tensor_dict
549
+
550
+ group = self.device_group
551
+ metadata_group = self.cpu_group
552
+
553
+ if dst is None:
554
+ dst = self.next_rank
555
+ assert dst < self.world_size, f"Invalid dst rank ({dst})"
556
+
557
+ metadata_list: List[Tuple[Any, Any]] = []
558
+ assert isinstance(
559
+ tensor_dict,
560
+ dict), f"Expecting a dictionary, got {type(tensor_dict)}"
561
+ metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
562
+ # `metadata_list` lives in CPU memory.
563
+ # `send_object_list` has serialization & deserialization,
564
+ # all happening on CPU. Therefore, we can use the CPU group.
565
+ self.send_object(metadata_list, dst=dst)
566
+ for tensor in tensor_list:
567
+ if tensor.numel() == 0:
568
+ # Skip sending empty tensors.
569
+ continue
570
+ if tensor.is_cpu:
571
+ # use metadata_group for CPU tensors
572
+ torch.distributed.send(tensor, dst=dst, group=metadata_group)
573
+ else:
574
+ # use group for GPU tensors
575
+ torch.distributed.send(tensor, dst=dst, group=group)
576
+ return None
577
+
578
+ def recv_tensor_dict(
579
+ self,
580
+ src: Optional[int] = None
581
+ ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
582
+ """Recv the input tensor dictionary.
583
+ NOTE: `src` is the local rank of the source rank.
584
+ """
585
+ # Bypass the function if we are using only 1 GPU.
586
+ if not torch.distributed.is_initialized() or self.world_size == 1:
587
+ return None
588
+
589
+ group = self.device_group
590
+ metadata_group = self.cpu_group
591
+
592
+ if src is None:
593
+ src = self.prev_rank
594
+ assert src < self.world_size, f"Invalid src rank ({src})"
595
+
596
+ recv_metadata_list = self.recv_object(src=src)
597
+ tensor_dict = {}
598
+ for key, value in recv_metadata_list:
599
+ if isinstance(value, TensorMetadata):
600
+ tensor = torch.empty(value.size,
601
+ dtype=value.dtype,
602
+ device=value.device)
603
+ if tensor.numel() == 0:
604
+ # Skip broadcasting empty tensors.
605
+ tensor_dict[key] = tensor
606
+ continue
607
+ if tensor.is_cpu:
608
+ # use metadata_group for CPU tensors
609
+ torch.distributed.recv(tensor,
610
+ src=src,
611
+ group=metadata_group)
612
+ else:
613
+ # use group for GPU tensors
614
+ torch.distributed.recv(tensor, src=src, group=group)
615
+ tensor_dict[key] = tensor
616
+ else:
617
+ tensor_dict[key] = value
618
+ return tensor_dict
619
+
620
def barrier(self):
621
"""Barrier synchronization among the group.
622
NOTE: don't use `device_group` here! `barrier` in NCCL is
@@ -468,6 +626,35 @@ class GroupCoordinator:
626
"""
627
torch.distributed.barrier(group=self.cpu_group)
628
629
+ def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
630
+ """Sends a tensor to the destination rank in a non-blocking way"""
631
+ """NOTE: `dst` is the local rank of the destination rank."""
632
+ if dst is None:
633
+ dst = self.next_rank
634
+
635
+ pynccl_comm = self.pynccl_comm
636
+ if pynccl_comm is not None and not pynccl_comm.disabled:
637
+ pynccl_comm.send(tensor, dst)
638
+ else:
639
+ torch.distributed.send(tensor, self.ranks[dst], self.device_group)
640
+
641
+ def recv(self,
642
+ size: torch.Size,
643
+ dtype: torch.dtype,
644
+ src: Optional[int] = None) -> torch.Tensor:
645
+ """Receives a tensor from the src rank."""
646
+ """NOTE: `src` is the local rank of the destination rank."""
647
+ if src is None:
648
+ src = self.prev_rank
649
+
650
+ tensor = torch.empty(size, dtype=dtype, device=self.device)
651
+ pynccl_comm = self.pynccl_comm
652
+ if pynccl_comm is not None and not pynccl_comm.disabled:
653
+ pynccl_comm.recv(tensor, src)
654
+ else:
655
+ torch.distributed.recv(tensor, self.ranks[src], self.device_group)
656
+ return tensor
657
+
658
def destroy(self):
659
if self.device_group is not None:
660
torch.distributed.destroy_process_group(self.device_group)