A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[VLM][BugFix] Make sure that `multi_modal_kwargs` can broadcast properly with ring buffer. (#5905)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
xwjiang2010 committed
Jun 28, 2024 at 00:29 UTC
74d55c065b104f816fca9c177e044415802796a1
1 file changed
+10
-10
vllm/distributed/parallel_state.py
+10
-10
@@ -45,7 +45,7 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
45
46
47
def _split_tensor_dict(
48
- tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
48
+ tensor_dict: Dict[str, Union[torch.Tensor, Any]],
49
prefix: str = "") -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
50
"""Split the tensor dictionary into two parts:
51
1. A list of (key, value) pairs. If the value is a tensor, it is replaced
@@ -473,11 +473,11 @@ class GroupCoordinator:
473
474
def broadcast_tensor_dict(
475
self,
476
- tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
476
+ tensor_dict: Optional[Dict[str, Union[torch.Tensor, Any]]] = None,
477
src: int = 0,
478
group: Optional[ProcessGroup] = None,
479
metadata_group: Optional[ProcessGroup] = None
480
- ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
480
+ ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
481
"""Broadcast the input tensor dictionary.
482
NOTE: `src` is the local rank of the source rank.
483
"""
@@ -558,9 +558,9 @@ class GroupCoordinator:
558
559
def send_tensor_dict(
560
self,
561
- tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
561
+ tensor_dict: Dict[str, Union[torch.Tensor, Any]],
562
dst: Optional[int] = None
563
- ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
563
+ ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
564
"""Send the input tensor dictionary.
565
NOTE: `dst` is the local rank of the source rank.
566
"""
@@ -599,7 +599,7 @@ class GroupCoordinator:
599
def recv_tensor_dict(
600
self,
601
src: Optional[int] = None
602
- ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
602
+ ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
603
"""Recv the input tensor dictionary.
604
NOTE: `src` is the local rank of the source rank.
605
"""
@@ -615,7 +615,7 @@ class GroupCoordinator:
615
assert src < self.world_size, f"Invalid src rank ({src})"
616
617
recv_metadata_list = self.recv_object(src=src)
618
- tensor_dict = {}
618
+ tensor_dict: Dict[str, Any] = {}
619
for key, value in recv_metadata_list:
620
if isinstance(value, TensorMetadata):
621
tensor = torch.empty(value.size,
@@ -623,7 +623,7 @@ class GroupCoordinator:
623
device=value.device)
624
if tensor.numel() == 0:
625
# Skip broadcasting empty tensors.
626
- tensor_dict[key] = tensor
626
+ _update_nested_dict(tensor_dict, key, tensor)
627
continue
628
if tensor.is_cpu:
629
# use metadata_group for CPU tensors
@@ -633,9 +633,9 @@ class GroupCoordinator:
633
else:
634
# use group for GPU tensors
635
torch.distributed.recv(tensor, src=src, group=group)
636
- tensor_dict[key] = tensor
636
+ _update_nested_dict(tensor_dict, key, tensor)
637
else:
638
- tensor_dict[key] = value
638
+ _update_nested_dict(tensor_dict, key, value)
639
return tensor_dict
640
641
def barrier(self):