A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[VLM][Bugfix] Make sure that `multi_modal_kwargs` is broadcasted properly (#5880)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com>
xwjiang2010 committed
Jun 27, 2024 at 00:15 UTC
d12af207d24fda5ba6cea08caa1073f9d4413938
3 files changed
+81
-9
.buildkite/test-pipeline.yaml
+3
-1
@@ -27,7 +27,9 @@ steps:
27
28
- label: Core Test
29
mirror_hardwares: [amd]
30
- command: pytest -v -s core
30
+ commands:
31
+ - pytest -v -s core
32
+ - pytest -v -s distributed/test_parallel_state.py
33
34
- label: Distributed Comm Ops Test
35
#mirror_hardwares: [amd]
tests/distributed/test_parallel_state.py
new
+49
@@ -0,0 +1,49 @@
1
+from typing import Any, Dict
2
+
3
+import torch
4
+
5
+from vllm.distributed.parallel_state import (_split_tensor_dict,
6
+ _update_nested_dict)
7
+
8
+
9
+def test_split_tensor_dict():
10
+ test_dict = {
11
+ "key_a": "a",
12
+ "key_b": torch.arange(8, dtype=torch.float32),
13
+ "key_c": {
14
+ "key_1": torch.arange(5, dtype=torch.float32),
15
+ "key_2": torch.tensor([], dtype=torch.float32),
16
+ "key_3": 123,
17
+ },
18
+ "key_d": {},
19
+ }
20
+ metadata_list, tensor_list = _split_tensor_dict(test_dict)
21
+ assert len(metadata_list) == 6
22
+ assert torch.allclose(tensor_list[0], test_dict["key_b"])
23
+ assert torch.allclose(tensor_list[1], test_dict["key_c"]["key_1"])
24
+ assert torch.allclose(tensor_list[2], test_dict["key_c"]["key_2"])
25
+
26
+
27
+def test_update_nested_dict():
28
+ flattened_keys_values = [("key1%key2%key3", "value1"),
29
+ ("key1%key2%key4", "value2"),
30
+ ("key1%key5", "value3"), ("key6%key7", "value4"),
31
+ ("key8", "value5")]
32
+ res: Dict[str, Any] = {}
33
+
34
+ # Update the nested dictionary with each flattened key-value pair
35
+ for flat_key, value in flattened_keys_values:
36
+ _update_nested_dict(res, flat_key, value)
37
+ assert res == {
38
+ "key1": {
39
+ "key2": {
40
+ "key3": "value1",
41
+ "key4": "value2"
42
+ },
43
+ "key5": "value3"
44
+ },
45
+ "key6": {
46
+ "key7": "value4"
47
+ },
48
+ "key8": "value5"
49
+ }
vllm/distributed/parallel_state.py
+29
-8
@@ -45,14 +45,17 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
45
46
47
def _split_tensor_dict(
48
- tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
49
-) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
48
+ tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
49
+ prefix: str = "") -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
50
"""Split the tensor dictionary into two parts:
51
1. A list of (key, value) pairs. If the value is a tensor, it is replaced
52
by its metadata.
53
2. A list of tensors.
54
+
55
+ If the Tensor is nested under `tensor_dict["key1"]["key2"]`, the key of its
56
+ metadata will be "key1%key2".
57
"""
55
- metadata_list = []
58
+ metadata_list: List[Tuple[str, Any]] = []
59
tensor_list = []
60
for key, value in tensor_dict.items():
61
if isinstance(value, torch.Tensor):
@@ -62,13 +65,31 @@ def _split_tensor_dict(
65
# receiving side will set the device index.
66
device = value.device.type
67
metadata_list.append(
65
- (key, TensorMetadata(device, value.dtype, value.size())))
68
+ (prefix + key, TensorMetadata(device, value.dtype,
69
+ value.size())))
70
tensor_list.append(value)
71
+ elif isinstance(value, dict):
72
+ if len(value) == 0:
73
+ metadata_list.append((prefix + key, value))
74
+ inner_metadata_list, inner_tensor_list = _split_tensor_dict(
75
+ value, prefix + key + "%")
76
+ metadata_list.extend(inner_metadata_list)
77
+ tensor_list.extend(inner_tensor_list)
78
else:
68
- metadata_list.append((key, value))
79
+ metadata_list.append((prefix + key, value))
80
return metadata_list, tensor_list
81
82
83
+def _update_nested_dict(nested_dict, flattened_key, value):
84
+ key_splits = flattened_key.split("%")
85
+ cur_dict = nested_dict
86
+ for k in key_splits[:-1]:
87
+ if k not in cur_dict:
88
+ cur_dict[k] = {}
89
+ cur_dict = cur_dict[k]
90
+ cur_dict[key_splits[-1]] = value
91
+
92
+
93
class GroupCoordinator:
94
"""
95
PyTorch ProcessGroup wrapper for a group of processes.
@@ -512,7 +533,7 @@ class GroupCoordinator:
533
device=value.device)
534
if tensor.numel() == 0:
535
# Skip broadcasting empty tensors.
515
- tensor_dict[key] = tensor
536
+ _update_nested_dict(tensor_dict, key, tensor)
537
continue
538
if tensor.is_cpu:
539
# use metadata_group for CPU tensors
@@ -528,9 +549,9 @@ class GroupCoordinator:
549
group=group,
550
async_op=True)
551
async_handles.append(handle)
531
- tensor_dict[key] = tensor
552
+ _update_nested_dict(tensor_dict, key, tensor)
553
else:
533
- tensor_dict[key] = value
554
+ _update_nested_dict(tensor_dict, key, value)
555
for async_handle in async_handles:
556
async_handle.wait()
557
return tensor_dict