@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef9b636e2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Simplify broadcast logic for control messages (#2501)

Zhuohan Li committed Jan 19, 2024 at 11:23 UTC ef9b636e2d427f588bf11242e312ba8954d9aff0
4 files changed +146 -129
tests/distributed/test_comm_ops.py
+33 -2
@@ -11,6 +11,7 @@ from vllm.utils import get_open_port
11 from vllm.model_executor.parallel_utils.communication_op import (
12 tensor_model_parallel_all_reduce,
13 tensor_model_parallel_all_gather,
14 + broadcast_tensor_dict,
15 )
16 from vllm.worker.worker import _init_distributed_environment
17
@@ -64,11 +65,41 @@ def all_gather_test_worker(tensor_parallel_size: int, rank: int,
65 assert torch.allclose(t, expected)
66
67
68 +@ray.remote(num_gpus=1, max_calls=1)
69 +def broadcast_tensor_dict_test_worker(tensor_parallel_size: int, rank: int,
70 + distributed_init_port: str):
71 + init_test_distributed_environment(1, tensor_parallel_size, rank,
72 + distributed_init_port)
73 + test_dict = {
74 + "a": torch.arange(8, dtype=torch.float32, device="cuda"),
75 + "b": torch.arange(16, dtype=torch.int8, device="cuda"),
76 + "c": "test",
77 + "d": [1, 2, 3],
78 + "e": {
79 + "a": 1,
80 + "b": 2
81 + },
82 + }
83 +
84 + if rank == 0:
85 + broadcast_tensor_dict(test_dict, src=0)
86 + else:
87 + recv_dict = broadcast_tensor_dict(src=0)
88 + assert len(recv_dict) == len(test_dict)
89 + assert torch.allclose(recv_dict["a"], test_dict["a"])
90 + assert torch.allclose(recv_dict["b"], test_dict["b"])
91 + assert recv_dict["c"] == test_dict["c"]
92 + assert recv_dict["d"] == test_dict["d"]
93 + assert recv_dict["e"] == test_dict["e"]
94 +
95 +
96 @pytest.mark.skipif(torch.cuda.device_count() < 2,
97 reason="Need at least 2 GPUs to run the test.")
98 @pytest.mark.parametrize("tensor_parallel_size", [2])
70 -@pytest.mark.parametrize("test_target",
71 - [all_reduce_test_worker, all_gather_test_worker])
99 +@pytest.mark.parametrize("test_target", [
100 + all_reduce_test_worker, all_gather_test_worker,
101 + broadcast_tensor_dict_test_worker
102 +])
103 def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
104 # Using ray helps debugging the error when it failed
105 # as compared to multiprocessing.
vllm/model_executor/parallel_utils/communication_op.py
+68 -5
@@ -1,3 +1,6 @@
1 +from collections import namedtuple
2 +from typing import Any, Dict, List, Optional, Union
3 +
4 import torch
5
6 from vllm.model_executor.parallel_utils.parallel_state import (
@@ -7,7 +10,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
10 )
11
12
10 -def tensor_model_parallel_all_reduce(input_):
13 +def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
14 """All-reduce the input tensor across model parallel group.
15
16 NOTE: This operation is applied in-place on the input tensor.
@@ -21,7 +24,8 @@ def tensor_model_parallel_all_reduce(input_):
24 return input_
25
26
24 -def tensor_model_parallel_all_gather(input_, dim=-1):
27 +def tensor_model_parallel_all_gather(input_: torch.Tensor,
28 + dim: int = -1) -> torch.Tensor:
29 """All-gather the input tensor across model parallel group."""
30 world_size = get_tensor_model_parallel_world_size()
31 # Bypass the function if we are using only 1 GPU.
@@ -48,7 +52,9 @@ def tensor_model_parallel_all_gather(input_, dim=-1):
52 return output_tensor
53
54
51 -def tensor_model_parallel_gather(input_, dst=0, dim=-1):
55 +def tensor_model_parallel_gather(input_: torch.Tensor,
56 + dst: int = 0,
57 + dim: int = -1) -> torch.Tensor:
58 """Gather the input tensor across model parallel group.
59
60 NOTE: We assume that the input tensor is on the same device across
@@ -80,7 +86,7 @@ def tensor_model_parallel_gather(input_, dst=0, dim=-1):
86 return output_tensor
87
88
83 -def broadcast(input_, src=0):
89 +def broadcast(input_: torch.Tensor, src: int = 0):
90 """Broadcast the input tensor."""
91 world_size = torch.distributed.get_world_size()
92 assert 0 <= src < world_size, f"Invalid src rank ({src})"
@@ -93,7 +99,7 @@ def broadcast(input_, src=0):
99 return input_
100
101
96 -def broadcast_object_list(obj_list, src=0):
102 +def broadcast_object_list(obj_list: List[Any], src: int = 0):
103 """Broadcast the input object list."""
104 world_size = torch.distributed.get_world_size()
105 assert 0 <= src < world_size, f"Invalid src rank ({src})"
@@ -104,3 +110,60 @@ def broadcast_object_list(obj_list, src=0):
110 # Broadcast.
111 torch.distributed.broadcast_object_list(obj_list, src=src)
112 return obj_list
113 +
114 +
115 +TensorMetadata = namedtuple("TensorMetadata", ["dtype", "size"])
116 +
117 +
118 +def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
119 + Any]]] = None,
120 + src: int = 0) -> Dict[Any, Union[torch.Tensor, Any]]:
121 + """Broadcast the input tensor dictionary."""
122 + rank = torch.distributed.get_rank()
123 + world_size = torch.distributed.get_world_size()
124 + assert 0 <= src < world_size, f"Invalid src rank ({src})"
125 +
126 + # Bypass the function if we are using only 1 GPU.
127 + if world_size == 1:
128 + return tensor_dict
129 +
130 + if rank == src:
131 + assert isinstance(
132 + tensor_dict,
133 + dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
134 + metadata_list = []
135 + for key, value in tensor_dict.items():
136 + if isinstance(value, torch.Tensor):
137 + assert value.is_cuda, (
138 + f"Tensor {key}: {value} is not on cuda. Currently we only "
139 + f"support broadcasting tensors on cuda.")
140 + metadata_list.append(
141 + (key, TensorMetadata(value.dtype, value.size())))
142 + else:
143 + metadata_list.append((key, value))
144 + torch.distributed.broadcast_object_list([metadata_list], src=src)
145 + for key, value in metadata_list:
146 + if isinstance(value, TensorMetadata):
147 + tensor = tensor_dict[key]
148 + torch.distributed.broadcast(tensor, src=src)
149 + else:
150 + recv_metadata_list = [None]
151 + torch.distributed.broadcast_object_list(recv_metadata_list, src=src)
152 + metadata_list = recv_metadata_list[0]
153 + tensor_dict = {}
154 + async_handles = []
155 + for key, value in metadata_list:
156 + if isinstance(value, TensorMetadata):
157 + tensor = torch.empty(value.size,
158 + dtype=value.dtype,
159 + device="cuda")
160 + async_handle = torch.distributed.broadcast(tensor,
161 + src=src,
162 + async_op=True)
163 + async_handles.append(async_handle)
164 + tensor_dict[key] = tensor
165 + else:
166 + tensor_dict[key] = value
167 + for async_handle in async_handles:
168 + async_handle.wait()
169 + return tensor_dict
vllm/worker/model_runner.py
+30 -108
@@ -9,7 +9,7 @@ from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 from vllm.model_executor.parallel_utils.communication_op import (
12 - broadcast, broadcast_object_list)
12 + broadcast_tensor_dict)
13 from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 from vllm.utils import in_wsl
@@ -393,121 +393,43 @@ class ModelRunner:
393 prompt_lens,
394 subquery_lens)
395
396 - def get_size_or_none(x: Optional[torch.Tensor]):
397 - return x.size() if x is not None else None
398 -
399 - # Broadcast the input data. For input tensors, we first broadcast
400 - # its shape and then broadcast the tensor to avoid high
401 - # serialization cost.
402 - py_data = {
403 - "input_tokens_size":
404 - input_tokens.size(),
405 - "input_positions_size":
406 - input_positions.size(),
407 - "is_prompt":
408 - input_metadata.is_prompt,
409 - "slot_mapping_size":
410 - get_size_or_none(input_metadata.slot_mapping),
411 - "prompt_lens_size":
412 - get_size_or_none(input_metadata.prompt_lens),
413 - "max_seq_len":
414 - input_metadata.max_seq_len,
415 - "start_loc_size":
416 - get_size_or_none(input_metadata.start_loc),
417 - "max_context_len":
418 - input_metadata.max_context_len,
419 - "context_lens_size":
420 - get_size_or_none(input_metadata.context_lens),
421 - "block_tables_size":
422 - get_size_or_none(input_metadata.block_tables),
423 - "use_cuda_graph":
424 - input_metadata.use_cuda_graph,
425 - "selected_token_indices_size":
426 - sampling_metadata.selected_token_indices.size(),
396 + # Broadcast the metadata.
397 + metadata_dict = {
398 + "input_tokens": input_tokens,
399 + "input_positions": input_positions,
400 + "is_prompt": input_metadata.is_prompt,
401 + "slot_mapping": input_metadata.slot_mapping,
402 + "prompt_lens": input_metadata.prompt_lens,
403 + "max_seq_len": input_metadata.max_seq_len,
404 + "start_loc": input_metadata.start_loc,
405 + "max_context_len": input_metadata.max_context_len,
406 + "context_lens": input_metadata.context_lens,
407 + "block_tables": input_metadata.block_tables,
408 + "use_cuda_graph": input_metadata.use_cuda_graph,
409 + "selected_token_indices":
410 + sampling_metadata.selected_token_indices,
411 }
428 - broadcast_object_list([py_data], src=0)
429 - # TODO(zhuohan): Combine the broadcasts or set async_op=True.
430 - broadcast(input_tokens, src=0)
431 - broadcast(input_positions, src=0)
432 - if input_metadata.slot_mapping is not None:
433 - broadcast(input_metadata.slot_mapping, src=0)
434 - if input_metadata.prompt_lens is not None:
435 - broadcast(input_metadata.prompt_lens, src=0)
436 - if input_metadata.start_loc is not None:
437 - broadcast(input_metadata.start_loc, src=0)
438 - if input_metadata.context_lens is not None:
439 - broadcast(input_metadata.context_lens, src=0)
440 - if input_metadata.block_tables is not None:
441 - broadcast(input_metadata.block_tables, src=0)
442 - broadcast(sampling_metadata.selected_token_indices, src=0)
412 + broadcast_tensor_dict(metadata_dict, src=0)
413 else:
444 - receving_list = [None]
445 - broadcast_object_list(receving_list, src=0)
446 - py_data = receving_list[0]
447 - input_tokens = torch.empty(*py_data["input_tokens_size"],
448 - dtype=torch.long,
449 - device="cuda")
450 - broadcast(input_tokens, src=0)
451 - input_positions = torch.empty(*py_data["input_positions_size"],
452 - dtype=torch.long,
453 - device="cuda")
454 - broadcast(input_positions, src=0)
455 - if py_data["slot_mapping_size"] is not None:
456 - slot_mapping = torch.empty(*py_data["slot_mapping_size"],
457 - dtype=torch.long,
458 - device="cuda")
459 - broadcast(slot_mapping, src=0)
460 - else:
461 - slot_mapping = None
462 - if py_data["prompt_lens_size"] is not None:
463 - prompt_lens = torch.empty(*py_data["prompt_lens_size"],
464 - dtype=torch.long,
465 - device="cuda")
466 - broadcast(prompt_lens, src=0)
467 - else:
468 - prompt_lens = None
469 - if py_data["start_loc_size"] is not None:
470 - start_loc = torch.empty(*py_data["start_loc_size"],
471 - dtype=torch.long,
472 - device="cuda")
473 - broadcast(start_loc, src=0)
474 - else:
475 - start_loc = None
476 - if py_data["context_lens_size"] is not None:
477 - context_lens = torch.empty(*py_data["context_lens_size"],
478 - dtype=torch.int,
479 - device="cuda")
480 - broadcast(context_lens, src=0)
481 - else:
482 - context_lens = None
483 - if py_data["block_tables_size"] is not None:
484 - block_tables = torch.empty(*py_data["block_tables_size"],
485 - dtype=torch.int,
486 - device="cuda")
487 - broadcast(block_tables, src=0)
488 - else:
489 - block_tables = None
490 - selected_token_indices = torch.empty(
491 - *py_data["selected_token_indices_size"],
492 - dtype=torch.long,
493 - device="cuda")
494 - broadcast(selected_token_indices, src=0)
414 + metadata_dict = broadcast_tensor_dict(src=0)
415 + input_tokens = metadata_dict["input_tokens"]
416 + input_positions = metadata_dict["input_positions"]
417 input_metadata = InputMetadata(
496 - is_prompt=py_data["is_prompt"],
497 - slot_mapping=slot_mapping,
498 - prompt_lens=prompt_lens,
499 - max_seq_len=py_data["max_seq_len"],
500 - start_loc=start_loc,
501 - max_context_len=py_data["max_context_len"],
502 - context_lens=context_lens,
503 - block_tables=block_tables,
504 - use_cuda_graph=py_data["use_cuda_graph"],
418 + is_prompt=metadata_dict["is_prompt"],
419 + slot_mapping=metadata_dict["slot_mapping"],
420 + prompt_lens=metadata_dict["prompt_lens"],
421 + max_seq_len=metadata_dict["max_seq_len"],
422 + start_loc=metadata_dict["start_loc"],
423 + max_context_len=metadata_dict["max_context_len"],
424 + context_lens=metadata_dict["context_lens"],
425 + block_tables=metadata_dict["block_tables"],
426 + use_cuda_graph=metadata_dict["use_cuda_graph"],
427 )
428 sampling_metadata = SamplingMetadata(
429 seq_groups=None,
430 seq_data=None,
431 prompt_lens=None,
510 - selected_token_indices=selected_token_indices,
432 + selected_token_indices=metadata_dict["selected_token_indices"],
433 categorized_sample_indices=None,
434 perform_sampling=False,
435 )
vllm/worker/worker.py
+15 -14
@@ -9,7 +9,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
10 from vllm.model_executor import set_random_seed
11 from vllm.model_executor.parallel_utils.communication_op import (
12 - broadcast_object_list)
12 + broadcast_tensor_dict)
13 from vllm.model_executor.parallel_utils.parallel_state import (
14 initialize_model_parallel)
15 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -175,20 +175,21 @@ class Worker:
175 assert blocks_to_swap_in is not None
176 assert blocks_to_swap_out is not None
177 assert blocks_to_copy is not None
178 - block_swapping_info = [
179 - blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy
180 - ]
181 - broadcast_object_list([num_seq_groups] + block_swapping_info,
182 - src=0)
178 + data = {
179 + "num_seq_groups": num_seq_groups,
180 + "blocks_to_swap_in": blocks_to_swap_in,
181 + "blocks_to_swap_out": blocks_to_swap_out,
182 + "blocks_to_copy": blocks_to_copy,
183 + }
184 + broadcast_tensor_dict(data, src=0)
185 else:
184 - # num_seq_groups, blocks_to_swap_in, blocks_to_swap_out,
185 - # blocks_to_copy (4 elements)
186 - recv_data = [None] * 4
187 - broadcast_object_list(recv_data, src=0)
188 - num_seq_groups = recv_data[0]
189 - block_swapping_info = recv_data[1:]
190 -
191 - self.cache_swap(*block_swapping_info)
186 + data = broadcast_tensor_dict(src=0)
187 + num_seq_groups = data["num_seq_groups"]
188 + blocks_to_swap_in = data["blocks_to_swap_in"]
189 + blocks_to_swap_out = data["blocks_to_swap_out"]
190 + blocks_to_copy = data["blocks_to_copy"]
191 +
192 + self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
193
194 # If there is no input, we don't need to execute the model.
195 if num_seq_groups == 0: