A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 10:45 UTC
[BugFix] Graceful handling of torch symm mem errors. (#27671)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Ilya Markov committed
Nov 12, 2025 at 01:41 UTC
1788aa1efb1f3cd8bf521885244aed3b89bed8a1
2 files changed
+17
-9
vllm/distributed/device_communicators/symm_mem.py
+15
-7
@@ -88,13 +88,21 @@ class SymmMemCommunicator:
88
self.max_size = SYMM_MEM_ALL_REDUCE_MAX_SIZES[self.device_capability][
89
self.world_size
90
]
91
-
92
- self.buffer = torch_symm_mem.empty(
93
- self.max_size // self.dtype.itemsize,
94
- device=self.device,
95
- dtype=self.dtype,
96
- )
97
- handle = torch_symm_mem.rendezvous(self.buffer, self.group.group_name)
91
+ try:
92
+ self.buffer = torch_symm_mem.empty(
93
+ self.max_size // self.dtype.itemsize,
94
+ device=self.device,
95
+ dtype=self.dtype,
96
+ )
97
+ handle = torch_symm_mem.rendezvous(self.buffer, self.group.group_name)
98
+ except RuntimeError as e:
99
+ logger.warning_once(
100
+ "SymmMemCommunicator: symmetric memory initialization failed: %s "
101
+ "Communicator is not available. To suppress this warning set "
102
+ "VLLM_ALLREDUCE_USE_SYMM_MEM=0",
103
+ str(e),
104
+ )
105
+ return
106
if handle.multicast_ptr == 0:
107
logger.warning(
108
"SymmMemCommunicator: symmetric memory "
vllm/envs.py
+2
-2
@@ -201,7 +201,7 @@ if TYPE_CHECKING:
201
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
202
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
203
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False
204
- VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False
204
+ VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
205
VLLM_TUNED_CONFIG_FOLDER: str | None = None
206
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
207
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
@@ -1389,7 +1389,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
1389
),
1390
# Whether to use pytorch symmetric memory for allreduce
1391
"VLLM_ALLREDUCE_USE_SYMM_MEM": lambda: bool(
1392
- int(os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"))
1392
+ int(os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "1"))
1393
),
1394
# Allows vllm to find tuned config under customized folder
1395
"VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),