A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 11:30 UTC
[All Reduce] Change default backend of Flashinfer All Reduce to trtllm (#35793)
Signed-off-by: hjjq <hanjieq@nvidia.com>
Hanjie Qiu committed
Mar 2, 2026 at 18:57 UTC
96fc09503a2dddde52cc0513bff9c08ebe664cbd
1 file changed
+5
-2
vllm/envs.py
+5
-2
@@ -168,7 +168,7 @@ if TYPE_CHECKING:
168
VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = (
169
"latency"
170
)
171
- VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
171
+ VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "trtllm"
172
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
173
VLLM_XGRAMMAR_CACHE_MB: int = 0
174
VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256
@@ -1297,9 +1297,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
1297
# Flashinfer fused allreduce backend.
1298
# "auto" will default to "mnnvl", which performs mostly same/better than "trtllm".
1299
# But "mnnvl" backend does not support fuse with quantization.
1300
+ # TODO: Default is "trtllm" right now because "mnnvl" has issues with cudagraph:
1301
+ # https://github.com/vllm-project/vllm/issues/35772
1302
+ # Should switch back to "auto" if the issue is resolved.
1303
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices(
1304
"VLLM_FLASHINFER_ALLREDUCE_BACKEND",
1302
- "auto",
1305
+ "trtllm",
1306
["auto", "trtllm", "mnnvl"],
1307
),
1308
# Control the workspace buffer size for the FlashInfer backend.