@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74d55c065b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[VLM][BugFix] Make sure that `multi_modal_kwargs` can broadcast properly with ring buffer. (#5905)

Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>

xwjiang2010 committed Jun 28, 2024 at 00:29 UTC 74d55c065b104f816fca9c177e044415802796a1
1 file changed +10 -10
vllm/distributed/parallel_state.py
+10 -10
@@ -45,7 +45,7 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
45
46
47 def _split_tensor_dict(
48 - tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
48 + tensor_dict: Dict[str, Union[torch.Tensor, Any]],
49 prefix: str = "") -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
50 """Split the tensor dictionary into two parts:
51 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
@@ -473,11 +473,11 @@ class GroupCoordinator:
473
474 def broadcast_tensor_dict(
475 self,
476 - tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
476 + tensor_dict: Optional[Dict[str, Union[torch.Tensor, Any]]] = None,
477 src: int = 0,
478 group: Optional[ProcessGroup] = None,
479 metadata_group: Optional[ProcessGroup] = None
480 - ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
480 + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
481 """Broadcast the input tensor dictionary.
482 NOTE: `src` is the local rank of the source rank.
483 """
@@ -558,9 +558,9 @@ class GroupCoordinator:
558
559 def send_tensor_dict(
560 self,
561 - tensor_dict: Dict[Any, Union[torch.Tensor, Any]],
561 + tensor_dict: Dict[str, Union[torch.Tensor, Any]],
562 dst: Optional[int] = None
563 - ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
563 + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
564 """Send the input tensor dictionary.
565 NOTE: `dst` is the local rank of the source rank.
566 """
@@ -599,7 +599,7 @@ class GroupCoordinator:
599 def recv_tensor_dict(
600 self,
601 src: Optional[int] = None
602 - ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
602 + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
603 """Recv the input tensor dictionary.
604 NOTE: `src` is the local rank of the source rank.
605 """
@@ -615,7 +615,7 @@ class GroupCoordinator:
615 assert src < self.world_size, f"Invalid src rank ({src})"
616
617 recv_metadata_list = self.recv_object(src=src)
618 - tensor_dict = {}
618 + tensor_dict: Dict[str, Any] = {}
619 for key, value in recv_metadata_list:
620 if isinstance(value, TensorMetadata):
621 tensor = torch.empty(value.size,
@@ -623,7 +623,7 @@ class GroupCoordinator:
623 device=value.device)
624 if tensor.numel() == 0:
625 # Skip broadcasting empty tensors.
626 - tensor_dict[key] = tensor
626 + _update_nested_dict(tensor_dict, key, tensor)
627 continue
628 if tensor.is_cpu:
629 # use metadata_group for CPU tensors
@@ -633,9 +633,9 @@ class GroupCoordinator:
633 else:
634 # use group for GPU tensors
635 torch.distributed.recv(tensor, src=src, group=group)
636 - tensor_dict[key] = tensor
636 + _update_nested_dict(tensor_dict, key, tensor)
637 else:
638 - tensor_dict[key] = value
638 + _update_nested_dict(tensor_dict, key, value)
639 return tensor_dict
640
641 def barrier(self):