A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Simplify broadcast logic for control messages (#2501)
Zhuohan Li committed
Jan 19, 2024 at 11:23 UTC
ef9b636e2d427f588bf11242e312ba8954d9aff0
4 files changed
+146
-129
tests/distributed/test_comm_ops.py
+33
-2
@@ -11,6 +11,7 @@ from vllm.utils import get_open_port
11
from vllm.model_executor.parallel_utils.communication_op import (
12
tensor_model_parallel_all_reduce,
13
tensor_model_parallel_all_gather,
14
+ broadcast_tensor_dict,
15
)
16
from vllm.worker.worker import _init_distributed_environment
17
@@ -64,11 +65,41 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
65
assert torch.allclose(t, expected)
66
67
68
+@ray.remote(num_gpus=1, max_calls=1)
69
+def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
70
+ distributed_init_port: str):
71
+ init_test_distributed_environment(1, tensor_parallel_size, rank,
72
+ distributed_init_port)
73
+ test_dict = {
74
+ "a": torch.arange(8, dtype=torch.float32, device="cuda"),
75
+ "b": torch.arange(16, dtype=torch.int8, device="cuda"),
76
+ "c": "test",
77
+ "d": [1, 2, 3],
78
+ "e": {
79
+ "a": 1,
80
+ "b": 2
81
+ },
82
+ }
83
+
84
+ if rank == 0:
85
+ broadcast_tensor_dict(test_dict, src=0)
86
+ else:
87
+ recv_dict = broadcast_tensor_dict(src=0)
88
+ assert len(recv_dict) == len(test_dict)
89
+ assert torch.allclose(recv_dict["a"], test_dict["a"])
90
+ assert torch.allclose(recv_dict["b"], test_dict["b"])
91
+ assert recv_dict["c"] == test_dict["c"]
92
+ assert recv_dict["d"] == test_dict["d"]
93
+ assert recv_dict["e"] == test_dict["e"]
94
+
95
+
96
@pytest.mark.skipif(torch.cuda.device_count() < 2,
97
reason="Need at least 2 GPUs to run the test.")
98
@pytest.mark.parametrize("tensor_parallel_size", [2])
70
-@pytest.mark.parametrize("test_target",
71
- [all_reduce_test_worker, all_gather_test_worker])
99
+@pytest.mark.parametrize("test_target", [
100
+ all_reduce_test_worker, all_gather_test_worker,
101
+ broadcast_tensor_dict_test_worker
102
+])
103
def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
104
# Using ray helps debugging the error when it failed
105
# as compared to multiprocessing.
vllm/model_executor/parallel_utils/communication_op.py
+68
-5
@@ -1,3 +1,6 @@
1
+from collections import namedtuple
2
+from typing import Any, Dict, List, Optional, Union
3
+
4
import torch
5
6
from vllm.model_executor.parallel_utils.parallel_state import (
@@ -7,7 +10,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
10
)
11
12
10
-def tensor_model_parallel_all_reduce(input_):
13
+def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
14
"""All-reduce the input tensor across model parallel group.
15
16
NOTE: This operation is applied in-place on the input tensor.
@@ -21,7 +24,8 @@ def tensor_model_parallel_all_reduce(input_):
24
return input_
25
26
24
-def tensor_model_parallel_all_gather(input_, dim=-1):
27
+def tensor_model_parallel_all_gather(input_: torch.Tensor,
28
+ dim: int = -1) -> torch.Tensor:
29
"""All-gather the input tensor across model parallel group."""
30
world_size = get_tensor_model_parallel_world_size()
31
# Bypass the function if we are using only 1 GPU.
@@ -48,7 +52,9 @@ def tensor_model_parallel_all_gather(input_, dim=-1):
52
return output_tensor
53
54
51
-def tensor_model_parallel_gather(input_, dst=0, dim=-1):
55
+def tensor_model_parallel_gather(input_: torch.Tensor,
56
+ dst: int = 0,
57
+ dim: int = -1) -> torch.Tensor:
58
"""Gather the input tensor across model parallel group.
59
60
NOTE: We assume that the input tensor is on the same device across
@@ -80,7 +86,7 @@ def tensor_model_parallel_gather(input_, dst=0, dim=-1):
86
return output_tensor
87
88
83
-def broadcast(input_, src=0):
89
+def broadcast(input_: torch.Tensor, src: int = 0):
90
"""Broadcast the input tensor."""
91
world_size = torch.distributed.get_world_size()
92
assert 0 <= src < world_size, f"Invalid src rank ({src})"
@@ -93,7 +99,7 @@ def broadcast(input_, src=0):
99
return input_
100
101
96
-def broadcast_object_list(obj_list, src=0):
102
+def broadcast_object_list(obj_list: List[Any], src: int = 0):
103
"""Broadcast the input object list."""
104
world_size = torch.distributed.get_world_size()
105
assert 0 <= src < world_size, f"Invalid src rank ({src})"
@@ -104,3 +110,60 @@ def broadcast_object_list(obj_list, src=0):
110
# Broadcast.
111
torch.distributed.broadcast_object_list(obj_list, src=src)
112
return obj_list
113
+
114
+
115
+TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
116
+
117
+
118
+def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
119
+ Any]]] = None,
120
+ src: int = 0) -> Dict[Any, Union[torch.Tensor, Any]]:
121
+ """Broadcast the input tensor dictionary."""
122
+ rank = torch.distributed.get_rank()
123
+ world_size = torch.distributed.get_world_size()
124
+ assert 0 <= src < world_size, f"Invalid src rank ({src})"
125
+
126
+ # Bypass the function if we are using only 1 GPU.
127
+ if world_size == 1:
128
+ return tensor_dict
129
+
130
+ if rank == src:
131
+ assert isinstance(
132
+ tensor_dict,
133
+ dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
134
+ metadata_list = []
135
+ for key, value in tensor_dict.items():
136
+ if isinstance(value, torch.Tensor):
137
+ assert value.is_cuda, (
138
+ f"Tensor {key}: {value} is not on cuda. Currently we only "
139
+ f"support broadcasting tensors on cuda.")
140
+ metadata_list.append(
141
+ (key, TensorMetadata(value.dtype, value.size())))
142
+ else:
143
+ metadata_list.append((key, value))
144
+ torch.distributed.broadcast_object_list([metadata_list], src=src)
145
+ for key, value in metadata_list:
146
+ if isinstance(value, TensorMetadata):
147
+ tensor = tensor_dict[key]
148
+ torch.distributed.broadcast(tensor, src=src)
149
+ else:
150
+ recv_metadata_list = [None]
151
+ torch.distributed.broadcast_object_list(recv_metadata_list, src=src)
152
+ metadata_list = recv_metadata_list[0]
153
+ tensor_dict = {}
154
+ async_handles = []
155
+ for key, value in metadata_list:
156
+ if isinstance(value, TensorMetadata):
157
+ tensor = torch.empty(value.size,
158
+ dtype=value.dtype,
159
+ device="cuda")
160
+ async_handle = torch.distributed.broadcast(tensor,
161
+ src=src,
162
+ async_op=True)
163
+ async_handles.append(async_handle)
164
+ tensor_dict[key] = tensor
165
+ else:
166
+ tensor_dict[key] = value
167
+ for async_handle in async_handles:
168
+ async_handle.wait()
169
+ return tensor_dict
vllm/worker/model_runner.py
+30
-108
@@ -9,7 +9,7 @@ from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
from vllm.model_executor.parallel_utils.communication_op import (
12
- broadcast, broadcast_object_list)
12
+ broadcast_tensor_dict)
13
from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
from vllm.utils import in_wsl
@@ -393,121 +393,43 @@ class ModelRunner:
393
prompt_lens,
394
subquery_lens)
395
396
- def get_size_or_none(x: Optional[torch.Tensor]):
397
- return x.size() if x is not None else None
398
-
399
- # Broadcast the input data. For input tensors, we first broadcast
400
- # its shape and then broadcast the tensor to avoid high
401
- # serialization cost.
402
- py_data = {
403
- "input_tokens_size":
404
- input_tokens.size(),
405
- "input_positions_size":
406
- input_positions.size(),
407
- "is_prompt":
408
- input_metadata.is_prompt,
409
- "slot_mapping_size":
410
- get_size_or_none(input_metadata.slot_mapping),
411
- "prompt_lens_size":
412
- get_size_or_none(input_metadata.prompt_lens),
413
- "max_seq_len":
414
- input_metadata.max_seq_len,
415
- "start_loc_size":
416
- get_size_or_none(input_metadata.start_loc),
417
- "max_context_len":
418
- input_metadata.max_context_len,
419
- "context_lens_size":
420
- get_size_or_none(input_metadata.context_lens),
421
- "block_tables_size":
422
- get_size_or_none(input_metadata.block_tables),
423
- "use_cuda_graph":
424
- input_metadata.use_cuda_graph,
425
- "selected_token_indices_size":
426
- sampling_metadata.selected_token_indices.size(),
396
+ # Broadcast the metadata.
397
+ metadata_dict = {
398
+ "input_tokens": input_tokens,
399
+ "input_positions": input_positions,
400
+ "is_prompt": input_metadata.is_prompt,
401
+ "slot_mapping": input_metadata.slot_mapping,
402
+ "prompt_lens": input_metadata.prompt_lens,
403
+ "max_seq_len": input_metadata.max_seq_len,
404
+ "start_loc": input_metadata.start_loc,
405
+ "max_context_len": input_metadata.max_context_len,
406
+ "context_lens": input_metadata.context_lens,
407
+ "block_tables": input_metadata.block_tables,
408
+ "use_cuda_graph": input_metadata.use_cuda_graph,
409
+ "selected_token_indices":
410
+ sampling_metadata.selected_token_indices,
411
}
428
- broadcast_object_list([py_data], src=0)
429
- # TODO(zhuohan): Combine the broadcasts or set async_op=True.
430
- broadcast(input_tokens, src=0)
431
- broadcast(input_positions, src=0)
432
- if input_metadata.slot_mapping is not None:
433
- broadcast(input_metadata.slot_mapping, src=0)
434
- if input_metadata.prompt_lens is not None:
435
- broadcast(input_metadata.prompt_lens, src=0)
436
- if input_metadata.start_loc is not None:
437
- broadcast(input_metadata.start_loc, src=0)
438
- if input_metadata.context_lens is not None:
439
- broadcast(input_metadata.context_lens, src=0)
440
- if input_metadata.block_tables is not None:
441
- broadcast(input_metadata.block_tables, src=0)
442
- broadcast(sampling_metadata.selected_token_indices, src=0)
412
+ broadcast_tensor_dict(metadata_dict, src=0)
413
else:
444
- receving_list = [None]
445
- broadcast_object_list(receving_list, src=0)
446
- py_data = receving_list[0]
447
- input_tokens = torch.empty(*py_data["input_tokens_size"],
448
- dtype=torch.long,
449
- device="cuda")
450
- broadcast(input_tokens, src=0)
451
- input_positions = torch.empty(*py_data["input_positions_size"],
452
- dtype=torch.long,
453
- device="cuda")
454
- broadcast(input_positions, src=0)
455
- if py_data["slot_mapping_size"] is not None:
456
- slot_mapping = torch.empty(*py_data["slot_mapping_size"],
457
- dtype=torch.long,
458
- device="cuda")
459
- broadcast(slot_mapping, src=0)
460
- else:
461
- slot_mapping = None
462
- if py_data["prompt_lens_size"] is not None:
463
- prompt_lens = torch.empty(*py_data["prompt_lens_size"],
464
- dtype=torch.long,
465
- device="cuda")
466
- broadcast(prompt_lens, src=0)
467
- else:
468
- prompt_lens = None
469
- if py_data["start_loc_size"] is not None:
470
- start_loc = torch.empty(*py_data["start_loc_size"],
471
- dtype=torch.long,
472
- device="cuda")
473
- broadcast(start_loc, src=0)
474
- else:
475
- start_loc = None
476
- if py_data["context_lens_size"] is not None:
477
- context_lens = torch.empty(*py_data["context_lens_size"],
478
- dtype=torch.int,
479
- device="cuda")
480
- broadcast(context_lens, src=0)
481
- else:
482
- context_lens = None
483
- if py_data["block_tables_size"] is not None:
484
- block_tables = torch.empty(*py_data["block_tables_size"],
485
- dtype=torch.int,
486
- device="cuda")
487
- broadcast(block_tables, src=0)
488
- else:
489
- block_tables = None
490
- selected_token_indices = torch.empty(
491
- *py_data["selected_token_indices_size"],
492
- dtype=torch.long,
493
- device="cuda")
494
- broadcast(selected_token_indices, src=0)
414
+ metadata_dict = broadcast_tensor_dict(src=0)
415
+ input_tokens = metadata_dict["input_tokens"]
416
+ input_positions = metadata_dict["input_positions"]
417
input_metadata = InputMetadata(
496
- is_prompt=py_data["is_prompt"],
497
- slot_mapping=slot_mapping,
498
- prompt_lens=prompt_lens,
499
- max_seq_len=py_data["max_seq_len"],
500
- start_loc=start_loc,
501
- max_context_len=py_data["max_context_len"],
502
- context_lens=context_lens,
503
- block_tables=block_tables,
504
- use_cuda_graph=py_data["use_cuda_graph"],
418
+ is_prompt=metadata_dict["is_prompt"],
419
+ slot_mapping=metadata_dict["slot_mapping"],
420
+ prompt_lens=metadata_dict["prompt_lens"],
421
+ max_seq_len=metadata_dict["max_seq_len"],
422
+ start_loc=metadata_dict["start_loc"],
423
+ max_context_len=metadata_dict["max_context_len"],
424
+ context_lens=metadata_dict["context_lens"],
425
+ block_tables=metadata_dict["block_tables"],
426
+ use_cuda_graph=metadata_dict["use_cuda_graph"],
427
)
428
sampling_metadata = SamplingMetadata(
429
seq_groups=None,
430
seq_data=None,
431
prompt_lens=None,
510
- selected_token_indices=selected_token_indices,
432
+ selected_token_indices=metadata_dict["selected_token_indices"],
433
categorized_sample_indices=None,
434
perform_sampling=False,
435
)
vllm/worker/worker.py
+15
-14
@@ -9,7 +9,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
10
from vllm.model_executor import set_random_seed
11
from vllm.model_executor.parallel_utils.communication_op import (
12
- broadcast_object_list)
12
+ broadcast_tensor_dict)
13
from vllm.model_executor.parallel_utils.parallel_state import (
14
initialize_model_parallel)
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -175,20 +175,21 @@ class Worker:
175
assert blocks_to_swap_in is not None
176
assert blocks_to_swap_out is not None
177
assert blocks_to_copy is not None
178
- block_swapping_info = [
179
- blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy
180
- ]
181
- broadcast_object_list([num_seq_groups] + block_swapping_info,
182
- src=0)
178
+ data = {
179
+ "num_seq_groups": num_seq_groups,
180
+ "blocks_to_swap_in": blocks_to_swap_in,
181
+ "blocks_to_swap_out": blocks_to_swap_out,
182
+ "blocks_to_copy": blocks_to_copy,
183
+ }
184
+ broadcast_tensor_dict(data, src=0)
185
else:
184
- # num_seq_groups, blocks_to_swap_in, blocks_to_swap_out,
185
- # blocks_to_copy (4 elements)
186
- recv_data = [None] * 4
187
- broadcast_object_list(recv_data, src=0)
188
- num_seq_groups = recv_data[0]
189
- block_swapping_info = recv_data[1:]
190
-
191
- self.cache_swap(*block_swapping_info)
186
+ data = broadcast_tensor_dict(src=0)
187
+ num_seq_groups = data["num_seq_groups"]
188
+ blocks_to_swap_in = data["blocks_to_swap_in"]
189
+ blocks_to_swap_out = data["blocks_to_swap_out"]
190
+ blocks_to_copy = data["blocks_to_copy"]
191
+
192
+ self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
193
194
# If there is no input, we don't need to execute the model.
195
if num_seq_groups == 0: