@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1d5402238
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix all-reduce memory usage (#2151)

Woosuk Kwon committed Dec 17, 2023 at 01:44 UTC e1d54022385ac52a3c3c6c6a3359d93f5c2944d5
1 file changed +8
vllm/worker/worker.py
+8
@@ -48,6 +48,14 @@ class Worker:
48 self.gpu_cache = None
49
50 def init_model(self, cupy_port: Optional[int] = None):
51 + # torch.distributed.all_reduce does not free the input tensor until
52 + # the synchronization point. This causes the memory usage to grow
53 + # as the number of all_reduce calls increases. This env var disables
54 + # this behavior.
55 + # Related issue:
56 + # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
57 + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
58 +
59 # This env var set by Ray causes exceptions with graph building.
60 os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
61 # Env vars will be set by Ray.