@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3711811b1d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Disable custom all reduce by default (#2808)

Woosuk Kwon committed Feb 8, 2024 at 09:58 UTC 3711811b1d2956e83e626c72f0e1607f2dfbc8fb
1 file changed +18 -8
vllm/config.py
+18 -8
@@ -388,16 +388,26 @@ class ParallelConfig:
388 if self.pipeline_parallel_size > 1:
389 raise NotImplementedError(
390 "Pipeline parallelism is not supported yet.")
391 - if is_hip():
391 + if not self.disable_custom_all_reduce and self.world_size > 1:
392 + if is_hip():
393 + self.disable_custom_all_reduce = True
394 + logger.info(
395 + "Disabled the custom all-reduce kernel because it is not "
396 + "supported on AMD GPUs.")
397 + elif self.pipeline_parallel_size > 1:
398 + self.disable_custom_all_reduce = True
399 + logger.info(
400 + "Disabled the custom all-reduce kernel because it is not "
401 + "supported with pipeline parallelism.")
402 +
403 + # FIXME(woosuk): Fix the stability issues and re-enable the custom
404 + # all-reduce kernel.
405 + if not self.disable_custom_all_reduce and self.world_size > 1:
406 self.disable_custom_all_reduce = True
407 logger.info(
394 - "Disabled the custom all-reduce kernel because it is not "
395 - "supported on AMD GPUs.")
396 - elif self.pipeline_parallel_size > 1:
397 - self.disable_custom_all_reduce = True
398 - logger.info(
399 - "Disabled the custom all-reduce kernel because it is not "
400 - "supported with pipeline parallelism.")
408 + "Custom all-reduce kernels are temporarily disabled due to "
409 + "stability issues. We will re-enable them once the issues are "
410 + "resolved.")
411
412
413 class SchedulerConfig: