A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 05:45 UTC
[Perf] Add SM 10.3 (B300/GB300) all-reduce communicator tuning (#37756)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
Mohammad Miadh Angkad committed
Mar 22, 2026 at 03:43 UTC
61e381dcf01f25cc8a006ecf0ba9c31dde662b42
3 files changed
+23
vllm/compilation/passes/fusion/allreduce_rms_fusion.py
+10
@@ -62,6 +62,11 @@ FI_ALLREDUCE_FUSION_MAX_SIZE_MB: dict[int, dict[int, float]] = {
62
4: 32, # 32MB
63
8: 1, # 1MB
64
},
65
+ 103: {
66
+ 2: 64, # 64MB
67
+ 4: 64, # 64MB
68
+ 8: 2, # 2MB
69
+ },
70
}
71
72
# Max size of the input tensor per world size per device capability
@@ -78,6 +83,11 @@ _FI_ALLREDUCE_ONE_SHOT_MAX_SIZES_MB: dict[int, dict[int, float]] = {
83
4: 4, # 4MB
84
8: 1, # 1MB
85
},
86
+ 103: {
87
+ 2: 32, # 32MB
88
+ 4: 4, # 4MB
89
+ 8: 2, # 2MB
90
+ },
91
}
92
93
vllm/distributed/device_communicators/all_reduce_utils.py
+12
@@ -44,6 +44,12 @@ CUSTOM_ALL_REDUCE_MAX_SIZES = {
44
6: 1 * MiB, # 1 MB
45
8: 1 * MiB, # 1 MB
46
},
47
+ "10.3": {
48
+ 2: 4 * MiB, # 4 MB
49
+ 4: 4 * MiB, # 4 MB
50
+ 6: 8 * MiB, # 8 MB
51
+ 8: 4 * MiB, # 4 MB
52
+ },
53
}
54
55
SYMM_MEM_ALL_REDUCE_MAX_SIZES = {
@@ -59,6 +65,12 @@ SYMM_MEM_ALL_REDUCE_MAX_SIZES = {
65
6: 128 * MiB, # 128 MB
66
8: 128 * MiB, # 128 MB
67
},
68
+ "10.3": {
69
+ 2: 4 * MiB, # 4 MB
70
+ 4: 32 * MiB, # 32 MB
71
+ 6: 32 * MiB, # 32 MB
72
+ 8: 64 * MiB, # 64 MB
73
+ },
74
}
75
76
# NCCL symmetric memory allreduce configuration based on H100 and GB200 benchmarks.
vllm/distributed/device_communicators/symm_mem.py
+1
@@ -28,6 +28,7 @@ class SymmMemCommunicator:
28
_WORLD_SIZES_MULTIMEM = {
29
"9.0": [4, 6, 8],
30
"10.0": [6, 8],
31
+ "10.3": [6, 8],
32
}
33
34
def __init__(