[Bugfix] batch_invariant: keep non-AllReduce collectives enabled on NCCL >= 2.31 (#58179)
Signed-off-by: Guanxin Li <38149783+guanxingithub@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
Guanxin Li committed
Sep 22, 2026 at 11:44 UTC
f944491db914880dc5f12c0b53c1aac113014a4b
1 file changed
+6
-1
vllm/model_executor/determinism/batch_invariant.py
+6
-1
@@ -1150,7 +1150,12 @@ def override_envs_for_invariance():
1150
os.environ["NCCL_MIN_NCHANNELS"] = "1"
1151
os.environ["NCCL_MAX_NCHANNELS"] = "1"
1152
os.environ["NCCL_PROTO"] = "Simple"
1153
- os.environ["NCCL_ALGO"] = "allreduce:tree"
1153
+ # NCCL >= 2.31 zero-fills the algorithm table of every collective when
1154
+ # NCCL_ALGO is set and re-enables only the named ones; together with the
1155
+ # NCCL_PROTO above, collectives not named here end up with no algorithm
1156
+ # and fail with ncclInvalidUsage. Re-enable Ring and Tree for all
1157
+ # collectives, then pin AllReduce to Tree for determinism.
1158
+ os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree"
1159
os.environ["NCCL_NTHREADS"] = "1"
1160
os.environ["NCCL_SOCKET_NTHREADS"] = "1"
1161