A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] use cpu group to broadcast metadata in cpu (#4444)
youkaichao committed
Apr 29, 2024 at 13:52 UTC
f4f921b7f12c67d3c4b7575caf5ddd9bd4b0b787
3 files changed
+63
-35
tests/tensorizer_loader/tensorize_vllm_model_for_testing.py
+3
-3
@@ -6,14 +6,14 @@ import uuid
6
from functools import partial
7
from typing import Type
8
9
-import torch
9
import torch.nn as nn
10
from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
11
TensorSerializer, stream_io)
12
from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
13
from transformers import AutoConfig, PretrainedConfig
14
16
-from vllm.distributed import initialize_model_parallel
15
+from vllm.distributed import (init_distributed_environment,
16
+ initialize_model_parallel)
17
from vllm.engine.arg_utils import EngineArgs
18
from vllm.engine.llm_engine import LLMEngine
19
from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
@@ -226,7 +226,7 @@ model_name = model_ref.split("/")[1]
226
os.environ["MASTER_ADDR"] = "127.0.0.1"
227
os.environ["MASTER_PORT"] = "8080"
228
229
-torch.distributed.init_process_group(world_size=1, rank=0)
229
+init_distributed_environment(world_size=1, rank=0, local_rank=0)
230
initialize_model_parallel()
231
232
keyfile = args.keyfile if args.keyfile else None
tests/worker/test_model_runner.py
+12
-11
@@ -2,8 +2,10 @@ import pytest
2
import torch
3
4
from vllm.config import ModelConfig, SchedulerConfig
5
+from vllm.distributed.parallel_state import init_distributed_environment
6
from vllm.model_executor.sampling_metadata import SamplingMetadata
7
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8
+from vllm.utils import get_open_port
9
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
10
11
@@ -249,19 +251,18 @@ def test_empty_seq_group():
251
assert len(return_prompt_lens) == 0
252
253
252
-@pytest.mark.parametrize("batch_size", list(range(2, 128)))
253
-@pytest.mark.parametrize("enforce_eager", [True, False])
254
-def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
255
-
256
- def get_world_size(group=None):
257
- return 1
254
+@pytest.fixture
255
+def distributed_init():
256
+ init_distributed_environment(
257
+ world_size=1,
258
+ rank=0,
259
+ distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
260
+ local_rank=0)
261
259
- def mock_get_process_group_ranks(group=None):
260
- return [0]
262
262
- monkeypatch.setattr(torch.distributed, "get_world_size", get_world_size)
263
- monkeypatch.setattr(torch.distributed, "get_process_group_ranks",
264
- mock_get_process_group_ranks)
263
+@pytest.mark.parametrize("batch_size", list(range(2, 128)))
264
+@pytest.mark.parametrize("enforce_eager", [True, False])
265
+def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
266
267
model_config = ModelConfig(
268
"facebook/opt-125m",
vllm/distributed/communication_op.py
+48
-21
@@ -4,7 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union
4
import torch
5
from torch.distributed import ProcessGroup
6
7
-from .parallel_state import (get_tensor_model_parallel_group,
7
+from .parallel_state import (get_cpu_world_group,
8
+ get_tensor_model_parallel_group,
9
get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size,
11
is_pynccl_enabled_for_all_reduce)
@@ -140,13 +141,46 @@ def broadcast_object_list(obj_list: List[Any],
141
TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
142
143
144
+def _split_tensor_dict(
145
+ tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
146
+) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
147
+ """Split the tensor dictionary into two parts:
148
+ 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
149
+ by its metadata.
150
+ 2. A list of tensors.
151
+ """
152
+ metadata_list = []
153
+ tensor_list = []
154
+ for key, value in tensor_dict.items():
155
+ if isinstance(value, torch.Tensor):
156
+ # Note(youkaichao): currently this only supports broadcasting
157
+ # tensors on cuda. In the future, we can add device as a field in
158
+ # TensorMetadata to support broadcasting tensors on different
159
+ # devices.
160
+ assert value.is_cuda, (
161
+ f"Tensor {key}: {value} is not on cuda. Currently we only "
162
+ f"support broadcasting tensors on cuda.")
163
+ metadata_list.append((key, TensorMetadata(value.dtype,
164
+ value.size())))
165
+ tensor_list.append(value)
166
+ else:
167
+ metadata_list.append((key, value))
168
+ return metadata_list, tensor_list
169
+
170
+
171
def broadcast_tensor_dict(
172
tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
173
src: int = 0,
174
group: Optional[ProcessGroup] = None,
175
+ metadata_group: Optional[ProcessGroup] = None
176
) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
148
- """Broadcast the input tensor dictionary."""
177
+ """Broadcast the input tensor dictionary.
178
+ `group` is used to broadcast the tensors, while `metadata_group` is used
179
+ to broadcast the metadata of the dict (e.g. dict structure, tensor sizes,
180
+ dtypes).
181
+ """
182
group = group or torch.distributed.group.WORLD
183
+ metadata_group = metadata_group or get_cpu_world_group()
184
ranks = torch.distributed.get_process_group_ranks(group)
185
assert src in ranks, f"Invalid src rank ({src})"
186
@@ -161,27 +195,20 @@ def broadcast_tensor_dict(
195
assert isinstance(
196
tensor_dict,
197
dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
164
- for key, value in tensor_dict.items():
165
- if isinstance(value, torch.Tensor):
166
- assert value.is_cuda, (
167
- f"Tensor {key}: {value} is not on cuda. Currently we only "
168
- f"support broadcasting tensors on cuda.")
169
- metadata_list.append(
170
- (key, TensorMetadata(value.dtype, value.size())))
171
- else:
172
- metadata_list.append((key, value))
198
+ metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
199
+ # `metadata_list` lives in CPU memory.
200
+ # `broadcast_object_list` involves serialization and deserialization,
201
+ # all happening on CPU. Therefore, we can use the CPU group.
202
torch.distributed.broadcast_object_list([metadata_list],
203
src=src,
175
- group=group)
204
+ group=metadata_group)
205
async_handles = []
177
- for key, value in metadata_list:
178
- if isinstance(value, TensorMetadata):
179
- tensor = tensor_dict[key]
180
- async_handles.append(
181
- torch.distributed.broadcast(tensor,
182
- src=src,
183
- group=group,
184
- async_op=True))
206
+ for tensor in tensor_list:
207
+ async_handles.append(
208
+ torch.distributed.broadcast(tensor,
209
+ src=src,
210
+ group=group,
211
+ async_op=True))
212
for async_handle in async_handles:
213
async_handle.wait()
214
@@ -189,7 +216,7 @@ def broadcast_tensor_dict(
216
recv_metadata_list = [None]
217
torch.distributed.broadcast_object_list(recv_metadata_list,
218
src=src,
192
- group=group)
219
+ group=metadata_group)
220
assert recv_metadata_list[0] is not None
221
tensor_dict = {}
222
async_handles = []