A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Misc] Do not override NCCL_CUMEM_ENABLE if set explicitly (#19105)
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com>
22quinn committed
Jun 5, 2025 at 04:14 UTC
188a4590d852324113a80cef6d0c237d6417a9d6
1 file changed
+12
-6
vllm/env_override.py
+12
-6
@@ -4,17 +4,23 @@ import os
4
5
import torch
6
7
+from vllm.logger import init_logger
8
+
9
+logger = init_logger(__name__)
10
+
11
# set some common config/environment variables that should be set
12
# for all processes created by vllm and all processes
13
# that interact with vllm workers.
14
# they are executed whenever `import vllm` is called.
15
12
-if not os.path.exists('/dev/nvidia-caps-imex-channels'):
13
- # normally, we disable NCCL_CUMEM_ENABLE because it
14
- # will cost 1~2 GiB GPU memory with cudagraph+allreduce,
15
- # see https://github.com/NVIDIA/nccl/issues/1234
16
- # for more details.
17
- # However, NCCL requires NCCL_CUMEM_ENABLE to work with
16
+if 'NCCL_CUMEM_ENABLE' in os.environ:
17
+ logger.warning(
18
+ "NCCL_CUMEM_ENABLE is set to %s, skipping override. "
19
+ "This may increase memory overhead with cudagraph+allreduce: "
20
+ "https://github.com/NVIDIA/nccl/issues/1234",
21
+ os.environ['NCCL_CUMEM_ENABLE'])
22
+elif not os.path.exists('/dev/nvidia-caps-imex-channels'):
23
+ # NCCL requires NCCL_CUMEM_ENABLE to work with
24
# multi-node NVLink, typically on GB200-NVL72 systems.
25
# The ultimate way to detect multi-node NVLink is to use
26
# NVML APIs, which are too expensive to call here.