@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b23c3f26f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add `group` as an argument in broadcast ops (#2522)

Junda Chen committed Jan 20, 2024 at 16:00 UTC 5b23c3f26fcf08fdbe2b7e6beecb4d970e632897
1 file changed +36 -17
vllm/model_executor/parallel_utils/communication_op.py
+36 -17
@@ -1,6 +1,8 @@
1 from collections import namedtuple
2 from typing import Any, Dict, List, Optional, Union
3
4 +from torch.distributed import ProcessGroup
5 +
6 import torch
7
8 from vllm.model_executor.parallel_utils.parallel_state import (
@@ -86,47 +88,59 @@ def tensor_model_parallel_gather(input_: torch.Tensor,
88 return output_tensor
89
90
89 -def broadcast(input_: torch.Tensor, src: int = 0):
91 +def broadcast(input_: torch.Tensor,
92 + src: int = 0,
93 + group: Optional[ProcessGroup] = None):
94 """Broadcast the input tensor."""
91 - world_size = torch.distributed.get_world_size()
92 - assert 0 <= src < world_size, f"Invalid src rank ({src})"
95 + group = group or torch.distributed.group.WORLD
96 + ranks = torch.distributed.get_process_group_ranks(group)
97 + assert src in ranks, f"Invalid src rank ({src})"
98
99 # Bypass the function if we are using only 1 GPU.
100 + world_size = torch.distributed.get_world_size(group=group)
101 if world_size == 1:
102 return input_
103 # Broadcast.
98 - torch.distributed.broadcast(input_, src=src)
104 + torch.distributed.broadcast(input_, src=src, group=group)
105 return input_
106
107
102 -def broadcast_object_list(obj_list: List[Any], src: int = 0):
108 +def broadcast_object_list(obj_list: List[Any],
109 + src: int = 0,
110 + group: Optional[ProcessGroup] = None):
111 """Broadcast the input object list."""
104 - world_size = torch.distributed.get_world_size()
105 - assert 0 <= src < world_size, f"Invalid src rank ({src})"
112 + group = group or torch.distributed.group.WORLD
113 + ranks = torch.distributed.get_process_group_ranks(group)
114 + assert src in ranks, f"Invalid src rank ({src})"
115
116 # Bypass the function if we are using only 1 GPU.
117 + world_size = torch.distributed.get_world_size(group=group)
118 if world_size == 1:
119 return obj_list
120 # Broadcast.
111 - torch.distributed.broadcast_object_list(obj_list, src=src)
121 + torch.distributed.broadcast_object_list(obj_list, src=src, group=group)
122 return obj_list
123
124
125 TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
126
127
118 -def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
119 - Any]]] = None,
120 - src: int = 0) -> Dict[Any, Union[torch.Tensor, Any]]:
128 +def broadcast_tensor_dict(
129 + tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
130 + src: int = 0,
131 + group: Optional[ProcessGroup] = None,
132 +) -> Dict[Any, Union[torch.Tensor, Any]]:
133 """Broadcast the input tensor dictionary."""
122 - rank = torch.distributed.get_rank()
123 - world_size = torch.distributed.get_world_size()
124 - assert 0 <= src < world_size, f"Invalid src rank ({src})"
134 + group = group or torch.distributed.group.WORLD
135 + ranks = torch.distributed.get_process_group_ranks(group)
136 + assert src in ranks, f"Invalid src rank ({src})"
137
138 # Bypass the function if we are using only 1 GPU.
139 + world_size = torch.distributed.get_world_size(group=group)
140 if world_size == 1:
141 return tensor_dict
142
143 + rank = torch.distributed.get_rank()
144 if rank == src:
145 assert isinstance(
146 tensor_dict,
@@ -141,14 +155,18 @@ def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
155 (key, TensorMetadata(value.dtype, value.size())))
156 else:
157 metadata_list.append((key, value))
144 - torch.distributed.broadcast_object_list([metadata_list], src=src)
158 + torch.distributed.broadcast_object_list([metadata_list],
159 + src=src,
160 + group=group)
161 for key, value in metadata_list:
162 if isinstance(value, TensorMetadata):
163 tensor = tensor_dict[key]
164 torch.distributed.broadcast(tensor, src=src)
165 else:
166 recv_metadata_list = [None]
151 - torch.distributed.broadcast_object_list(recv_metadata_list, src=src)
167 + torch.distributed.broadcast_object_list(recv_metadata_list,
168 + src=src,
169 + group=group)
170 metadata_list = recv_metadata_list[0]
171 tensor_dict = {}
172 async_handles = []
@@ -159,7 +177,8 @@ def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
177 device="cuda")
178 async_handle = torch.distributed.broadcast(tensor,
179 src=src,
162 - async_op=True)
180 + async_op=True,
181 + group=group)
182 async_handles.append(async_handle)
183 tensor_dict[key] = tensor
184 else: