A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix all-reduce memory usage (#2151)
Woosuk Kwon committed
Dec 17, 2023 at 01:44 UTC
e1d54022385ac52a3c3c6c6a3359d93f5c2944d5
1 file changed
+8
vllm/worker/worker.py
+8
@@ -48,6 +48,14 @@ class Worker:
48
self.gpu_cache = None
49
50
def init_model(self, cupy_port: Optional[int] = None):
51
+ # torch.distributed.all_reduce does not free the input tensor until
52
+ # the synchronization point. This causes the memory usage to grow
53
+ # as the number of all_reduce calls increases. This env var disables
54
+ # this behavior.
55
+ # Related issue:
56
+ # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
57
+ os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
58
+
59
# This env var set by Ray causes exceptions with graph building.
60
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
61
# Env vars will be set by Ray.