@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f4f921b7f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] use cpu group to broadcast metadata in cpu (#4444)

youkaichao committed Apr 29, 2024 at 13:52 UTC f4f921b7f12c67d3c4b7575caf5ddd9bd4b0b787
3 files changed +63 -35
tests/tensorizer_loader/tensorize_vllm_model_for_testing.py
+3 -3
@@ -6,14 +6,14 @@ import uuid
6 from functools import partial
7 from typing import Type
8
9 -import torch
9 import torch.nn as nn
10 from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
11 TensorSerializer, stream_io)
12 from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
13 from transformers import AutoConfig, PretrainedConfig
14
16 -from vllm.distributed import initialize_model_parallel
15 +from vllm.distributed import (init_distributed_environment,
16 + initialize_model_parallel)
17 from vllm.engine.arg_utils import EngineArgs
18 from vllm.engine.llm_engine import LLMEngine
19 from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
@@ -226,7 +226,7 @@ model_name = model_ref.split("/")[1]
226 os.environ["MASTER_ADDR"] = "127.0.0.1"
227 os.environ["MASTER_PORT"] = "8080"
228
229 -torch.distributed.init_process_group(world_size=1, rank=0)
229 +init_distributed_environment(world_size=1, rank=0, local_rank=0)
230 initialize_model_parallel()
231
232 keyfile = args.keyfile if args.keyfile else None
tests/worker/test_model_runner.py
+12 -11
@@ -2,8 +2,10 @@ import pytest
2 import torch
3
4 from vllm.config import ModelConfig, SchedulerConfig
5 +from vllm.distributed.parallel_state import init_distributed_environment
6 from vllm.model_executor.sampling_metadata import SamplingMetadata
7 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8 +from vllm.utils import get_open_port
9 from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
10
11
@@ -249,19 +251,18 @@ def test_empty_seq_group():
251 assert len(return_prompt_lens) == 0
252
253
252 -@pytest.mark.parametrize("batch_size", list(range(2, 128)))
253 -@pytest.mark.parametrize("enforce_eager", [True, False])
254 -def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
255 -
256 - def get_world_size(group=None):
257 - return 1
254 +@pytest.fixture
255 +def distributed_init():
256 + init_distributed_environment(
257 + world_size=1,
258 + rank=0,
259 + distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
260 + local_rank=0)
261
259 - def mock_get_process_group_ranks(group=None):
260 - return [0]
262
262 - monkeypatch.setattr(torch.distributed, "get_world_size", get_world_size)
263 - monkeypatch.setattr(torch.distributed, "get_process_group_ranks",
264 - mock_get_process_group_ranks)
263 +@pytest.mark.parametrize("batch_size", list(range(2, 128)))
264 +@pytest.mark.parametrize("enforce_eager", [True, False])
265 +def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
266
267 model_config = ModelConfig(
268 "facebook/opt-125m",
vllm/distributed/communication_op.py
+48 -21
@@ -4,7 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union
4 import torch
5 from torch.distributed import ProcessGroup
6
7 -from .parallel_state import (get_tensor_model_parallel_group,
7 +from .parallel_state import (get_cpu_world_group,
8 + get_tensor_model_parallel_group,
9 get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size,
11 is_pynccl_enabled_for_all_reduce)
@@ -140,13 +141,46 @@ def broadcast_object_list(obj_list: List[Any],
141 TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
142
143
144 +def _split_tensor_dict(
145 + tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
146 +) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
147 + """Split the tensor dictionary into two parts:
148 + 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
149 + by its metadata.
150 + 2. A list of tensors.
151 + """
152 + metadata_list = []
153 + tensor_list = []
154 + for key, value in tensor_dict.items():
155 + if isinstance(value, torch.Tensor):
156 + # Note(youkaichao): currently this only supports broadcasting
157 + # tensors on cuda. In the future, we can add device as a field in
158 + # TensorMetadata to support broadcasting tensors on different
159 + # devices.
160 + assert value.is_cuda, (
161 + f"Tensor {key}: {value} is not on cuda. Currently we only "
162 + f"support broadcasting tensors on cuda.")
163 + metadata_list.append((key, TensorMetadata(value.dtype,
164 + value.size())))
165 + tensor_list.append(value)
166 + else:
167 + metadata_list.append((key, value))
168 + return metadata_list, tensor_list
169 +
170 +
171 def broadcast_tensor_dict(
172 tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
173 src: int = 0,
174 group: Optional[ProcessGroup] = None,
175 + metadata_group: Optional[ProcessGroup] = None
176 ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
148 - """Broadcast the input tensor dictionary."""
177 + """Broadcast the input tensor dictionary.
178 + `group` is used to broadcast the tensors, while `metadata_group` is used
179 + to broadcast the metadata of the dict (e.g. dict structure, tensor sizes,
180 + dtypes).
181 + """
182 group = group or torch.distributed.group.WORLD
183 + metadata_group = metadata_group or get_cpu_world_group()
184 ranks = torch.distributed.get_process_group_ranks(group)
185 assert src in ranks, f"Invalid src rank ({src})"
186
@@ -161,27 +195,20 @@ def broadcast_tensor_dict(
195 assert isinstance(
196 tensor_dict,
197 dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
164 - for key, value in tensor_dict.items():
165 - if isinstance(value, torch.Tensor):
166 - assert value.is_cuda, (
167 - f"Tensor {key}: {value} is not on cuda. Currently we only "
168 - f"support broadcasting tensors on cuda.")
169 - metadata_list.append(
170 - (key, TensorMetadata(value.dtype, value.size())))
171 - else:
172 - metadata_list.append((key, value))
198 + metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
199 + # `metadata_list` lives in CPU memory.
200 + # `broadcast_object_list` involves serialization and deserialization,
201 + # all happening on CPU. Therefore, we can use the CPU group.
202 torch.distributed.broadcast_object_list([metadata_list],
203 src=src,
175 - group=group)
204 + group=metadata_group)
205 async_handles = []
177 - for key, value in metadata_list:
178 - if isinstance(value, TensorMetadata):
179 - tensor = tensor_dict[key]
180 - async_handles.append(
181 - torch.distributed.broadcast(tensor,
182 - src=src,
183 - group=group,
184 - async_op=True))
206 + for tensor in tensor_list:
207 + async_handles.append(
208 + torch.distributed.broadcast(tensor,
209 + src=src,
210 + group=group,
211 + async_op=True))
212 for async_handle in async_handles:
213 async_handle.wait()
214
@@ -189,7 +216,7 @@ def broadcast_tensor_dict(
216 recv_metadata_list = [None]
217 torch.distributed.broadcast_object_list(recv_metadata_list,
218 src=src,
192 - group=group)
219 + group=metadata_group)
220 assert recv_metadata_list[0] is not None
221 tensor_dict = {}
222 async_handles = []