@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc466a3290
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] support cpu&device in broadcast tensor dict (#4660)

[Core][Distributed] support both cpu and device tensor in broadcast tensor dict (#4660)

youkaichao committed May 7, 2024 at 19:34 UTC cc466a32903d53d0ceca459b766d74ad668c8f87
2 files changed +41 -22
tests/distributed/test_comm_ops.py
+6 -1
@@ -77,14 +77,18 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
77 init_test_distributed_environment(1, tensor_parallel_size, rank,
78 distributed_init_port)
79 test_dict = {
80 + # device tensor
81 "a": torch.arange(8, dtype=torch.float32, device="cuda"),
81 - "b": torch.arange(16, dtype=torch.int8, device="cuda"),
82 + # CPU tensor
83 + "b": torch.arange(16, dtype=torch.int8, device="cpu"),
84 "c": "test",
85 "d": [1, 2, 3],
86 "e": {
87 "a": 1,
88 "b": 2
89 },
90 + # empty tensor
91 + "f": torch.tensor([], dtype=torch.float32, device="cuda"),
92 }
93
94 if rank == 0:
@@ -97,6 +101,7 @@ def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
101 assert recv_dict["c"] == test_dict["c"]
102 assert recv_dict["d"] == test_dict["d"]
103 assert recv_dict["e"] == test_dict["e"]
104 + assert torch.allclose(recv_dict["f"], test_dict["f"])
105
106
107 @pytest.mark.skipif(torch.cuda.device_count() < 2,
vllm/distributed/communication_op.py
+35 -21
@@ -137,7 +137,7 @@ def broadcast_object_list(obj_list: List[Any],
137 return obj_list
138
139
140 -TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
140 +TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
141
142
143 def _split_tensor_dict(
@@ -152,15 +152,13 @@ def _split_tensor_dict(
152 tensor_list = []
153 for key, value in tensor_dict.items():
154 if isinstance(value, torch.Tensor):
155 - # Note(youkaichao): currently this only supports broadcasting
156 - # tensors on cuda. In the future, we can add device as a field in
157 - # TensorMetadata to support broadcasting tensors on different
158 - # devices.
159 - assert value.is_cuda, (
160 - f"Tensor {key}: {value} is not on cuda. Currently we only "
161 - f"support broadcasting tensors on cuda.")
162 - metadata_list.append((key, TensorMetadata(value.dtype,
163 - value.size())))
155 + # Note: we cannot use `value.device` here,
156 + # because it contains not only the device type but also the device
157 + # index (e.g. "cuda:0"). We only need the device type.
158 + # receiving side will set the device index.
159 + device = "cpu" if value.is_cpu else "cuda"
160 + metadata_list.append(
161 + (key, TensorMetadata(device, value.dtype, value.size())))
162 tensor_list.append(value)
163 else:
164 metadata_list.append((key, value))
@@ -206,11 +204,19 @@ def broadcast_tensor_dict(
204 if tensor.numel() == 0:
205 # Skip broadcasting empty tensors.
206 continue
209 - async_handles.append(
210 - torch.distributed.broadcast(tensor,
211 - src=src,
212 - group=group,
213 - async_op=True))
207 + if tensor.is_cpu:
208 + # use metadata_group for CPU tensors
209 + handle = torch.distributed.broadcast(tensor,
210 + src=src,
211 + group=metadata_group,
212 + async_op=True)
213 + else:
214 + # use group for GPU tensors
215 + handle = torch.distributed.broadcast(tensor,
216 + src=src,
217 + group=group,
218 + async_op=True)
219 + async_handles.append(handle)
220 for async_handle in async_handles:
221 async_handle.wait()
222
@@ -226,16 +232,24 @@ def broadcast_tensor_dict(
232 if isinstance(value, TensorMetadata):
233 tensor = torch.empty(value.size,
234 dtype=value.dtype,
229 - device="cuda")
235 + device=value.device)
236 if tensor.numel() == 0:
237 # Skip broadcasting empty tensors.
238 tensor_dict[key] = tensor
239 continue
234 - async_handle = torch.distributed.broadcast(tensor,
235 - src=src,
236 - async_op=True,
237 - group=group)
238 - async_handles.append(async_handle)
240 + if tensor.is_cpu:
241 + # use metadata_group for CPU tensors
242 + handle = torch.distributed.broadcast(tensor,
243 + src=src,
244 + group=metadata_group,
245 + async_op=True)
246 + else:
247 + # use group for GPU tensors
248 + handle = torch.distributed.broadcast(tensor,
249 + src=src,
250 + group=group,
251 + async_op=True)
252 + async_handles.append(handle)
253 tensor_dict[key] = tensor
254 else:
255 tensor_dict[key] = value