A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add `group` as an argument in broadcast ops (#2522)
Junda Chen committed
Jan 20, 2024 at 16:00 UTC
5b23c3f26fcf08fdbe2b7e6beecb4d970e632897
1 file changed
+36
-17
vllm/model_executor/parallel_utils/communication_op.py
+36
-17
@@ -1,6 +1,8 @@
1
from collections import namedtuple
2
from typing import Any, Dict, List, Optional, Union
3
4
+from torch.distributed import ProcessGroup
5
+
6
import torch
7
8
from vllm.model_executor.parallel_utils.parallel_state import (
@@ -86,47 +88,59 @@ def tensor_model_parallel_gather(input_: torch.Tensor,
88
return output_tensor
89
90
89
-def broadcast(input_: torch.Tensor, src: int = 0):
91
+def broadcast(input_: torch.Tensor,
92
+ src: int = 0,
93
+ group: Optional[ProcessGroup] = None):
94
"""Broadcast the input tensor."""
91
- world_size = torch.distributed.get_world_size()
92
- assert 0 <= src < world_size, f"Invalid src rank ({src})"
95
+ group = group or torch.distributed.group.WORLD
96
+ ranks = torch.distributed.get_process_group_ranks(group)
97
+ assert src in ranks, f"Invalid src rank ({src})"
98
99
# Bypass the function if we are using only 1 GPU.
100
+ world_size = torch.distributed.get_world_size(group=group)
101
if world_size == 1:
102
return input_
103
# Broadcast.
98
- torch.distributed.broadcast(input_, src=src)
104
+ torch.distributed.broadcast(input_, src=src, group=group)
105
return input_
106
107
102
-def broadcast_object_list(obj_list: List[Any], src: int = 0):
108
+def broadcast_object_list(obj_list: List[Any],
109
+ src: int = 0,
110
+ group: Optional[ProcessGroup] = None):
111
"""Broadcast the input object list."""
104
- world_size = torch.distributed.get_world_size()
105
- assert 0 <= src < world_size, f"Invalid src rank ({src})"
112
+ group = group or torch.distributed.group.WORLD
113
+ ranks = torch.distributed.get_process_group_ranks(group)
114
+ assert src in ranks, f"Invalid src rank ({src})"
115
116
# Bypass the function if we are using only 1 GPU.
117
+ world_size = torch.distributed.get_world_size(group=group)
118
if world_size == 1:
119
return obj_list
120
# Broadcast.
111
- torch.distributed.broadcast_object_list(obj_list, src=src)
121
+ torch.distributed.broadcast_object_list(obj_list, src=src, group=group)
122
return obj_list
123
124
125
TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
126
127
118
-def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
119
- Any]]] = None,
120
- src: int = 0) -> Dict[Any, Union[torch.Tensor, Any]]:
128
+def broadcast_tensor_dict(
129
+ tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
130
+ src: int = 0,
131
+ group: Optional[ProcessGroup] = None,
132
+) -> Dict[Any, Union[torch.Tensor, Any]]:
133
"""Broadcast the input tensor dictionary."""
122
- rank = torch.distributed.get_rank()
123
- world_size = torch.distributed.get_world_size()
124
- assert 0 <= src < world_size, f"Invalid src rank ({src})"
134
+ group = group or torch.distributed.group.WORLD
135
+ ranks = torch.distributed.get_process_group_ranks(group)
136
+ assert src in ranks, f"Invalid src rank ({src})"
137
138
# Bypass the function if we are using only 1 GPU.
139
+ world_size = torch.distributed.get_world_size(group=group)
140
if world_size == 1:
141
return tensor_dict
142
143
+ rank = torch.distributed.get_rank()
144
if rank == src:
145
assert isinstance(
146
tensor_dict,
@@ -141,14 +155,18 @@ def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
155
(key, TensorMetadata(value.dtype, value.size())))
156
else:
157
metadata_list.append((key, value))
144
- torch.distributed.broadcast_object_list([metadata_list], src=src)
158
+ torch.distributed.broadcast_object_list([metadata_list],
159
+ src=src,
160
+ group=group)
161
for key, value in metadata_list:
162
if isinstance(value, TensorMetadata):
163
tensor = tensor_dict[key]
164
torch.distributed.broadcast(tensor, src=src)
165
else:
166
recv_metadata_list = [None]
151
- torch.distributed.broadcast_object_list(recv_metadata_list, src=src)
167
+ torch.distributed.broadcast_object_list(recv_metadata_list,
168
+ src=src,
169
+ group=group)
170
metadata_list = recv_metadata_list[0]
171
tensor_dict = {}
172
async_handles = []
@@ -159,7 +177,8 @@ def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
177
device="cuda")
178
async_handle = torch.distributed.broadcast(tensor,
179
src=src,
162
- async_op=True)
180
+ async_op=True,
181
+ group=group)
182
async_handles.append(async_handle)
183
tensor_dict[key] = tensor
184
else: