@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4bf085ad0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[MISC] add support custom_op check (#8557)

Co-authored-by: youkaichao <youkaichao@126.com>

Kunshang Ji committed Sep 21, 2024 at 10:03 UTC d4bf085ad064ba68a77862e2022f37c33a66e94a
2 files changed +33 -22
vllm/distributed/parallel_state.py
+27 -22
@@ -36,6 +36,7 @@ from torch.distributed import Backend, ProcessGroup
36 import vllm.envs as envs
37 from vllm.logger import init_logger
38 from vllm.platforms import current_platform
39 +from vllm.utils import supports_custom_op
40
41
42 @dataclass
@@ -95,32 +96,33 @@ def _register_group(group: "GroupCoordinator") -> None:
96 _groups[group.unique_name] = weakref.ref(group) # type: ignore
97
98
98 -@torch.library.custom_op("vllm::inplace_all_reduce", mutates_args=["tensor"])
99 -def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None:
100 - assert group_name in _groups, f"Group {group_name} is not found."
101 - group = _groups[group_name]()
102 - if group is None:
103 - raise ValueError(f"Group {group_name} is destroyed.")
104 - group._all_reduce(tensor)
99 +if supports_custom_op():
100
101 + @torch.library.custom_op("vllm::inplace_all_reduce",
102 + mutates_args=["tensor"])
103 + def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None:
104 + assert group_name in _groups, f"Group {group_name} is not found."
105 + group = _groups[group_name]()
106 + if group is None:
107 + raise ValueError(f"Group {group_name} is destroyed.")
108 + group._all_reduce(tensor)
109
107 -@inplace_all_reduce.register_fake
108 -def _(tensor: torch.Tensor, group_name: str) -> None:
109 - return
110 -
111 -
112 -@torch.library.custom_op("vllm::outplace_all_reduce", mutates_args=[])
113 -def outplace_all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
114 - assert group_name in _groups, f"Group {group_name} is not found."
115 - group = _groups[group_name]()
116 - if group is None:
117 - raise ValueError(f"Group {group_name} is destroyed.")
118 - return group._all_reduce(tensor)
110 + @inplace_all_reduce.register_fake
111 + def _(tensor: torch.Tensor, group_name: str) -> None:
112 + return
113
114 + @torch.library.custom_op("vllm::outplace_all_reduce", mutates_args=[])
115 + def outplace_all_reduce(tensor: torch.Tensor,
116 + group_name: str) -> torch.Tensor:
117 + assert group_name in _groups, f"Group {group_name} is not found."
118 + group = _groups[group_name]()
119 + if group is None:
120 + raise ValueError(f"Group {group_name} is destroyed.")
121 + return group._all_reduce(tensor)
122
121 -@outplace_all_reduce.register_fake
122 -def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
123 - return torch.empty_like(tensor)
123 + @outplace_all_reduce.register_fake
124 + def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
125 + return torch.empty_like(tensor)
126
127
128 class GroupCoordinator:
@@ -335,6 +337,9 @@ class GroupCoordinator:
337 if self.world_size == 1:
338 return input_
339
340 + if not supports_custom_op():
341 + return self._all_reduce(input_)
342 +
343 if self.tpu_communicator is not None and \
344 not self.tpu_communicator.disabled:
345 # TPU handles Dynamo with its own logic.
vllm/utils.py
+6
@@ -1245,6 +1245,12 @@ def supports_dynamo() -> bool:
1245 return base_torch_version >= Version("2.4.0")
1246
1247
1248 +# Some backends use pytorch version < 2.4.0 which doesn't
1249 +# support `torch.library.custom_op`.
1250 +def supports_custom_op() -> bool:
1251 + return hasattr(torch.library, "custom_op")
1252 +
1253 +
1254 class AtomicCounter:
1255 """An atomic, thread-safe counter"""
1256