A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] Fix custom_ar support check (#7617)
bnellnm committed
Aug 16, 2024 at 22:05 UTC
e6803499941333e30e7db984cc590f21fc284e7b
1 file changed
+1
-1
vllm/distributed/device_communicators/custom_all_reduce.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.platforms import current_platform
15
from vllm.utils import cuda_device_count_stateless
16
17
try:
18
- assert ops.is_custom_op_supported("_C_custom_ar::meta_size")
18
+ ops.meta_size()
19
custom_ar = True
20
except Exception:
21
# For AMD GPUs and CPUs