A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 16:00 UTC
[ROCm] Enable VLLM triton FP8 moe for gfx1201, tuned for Qwen3-30B-A3B-FP8 tp=2 and Qwen/Qwen3.5-35B-A3B-FP8 tp=2 (#38086)
Signed-off-by: big-yellow-duck <jeffaw99@hotmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
vllmellm committed
Apr 2, 2026 at 16:13 UTC
551b3fb39f3a95ff3dc3feca9528ab4c90649316
4 files changed
+343
-4
vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=AMD_Radeon_R9700,dtype=fp8_w8a8,block_shape=[128,128].json
new
+165
@@ -0,0 +1,165 @@
1
+{
2
+ "triton_version": "3.6.0",
3
+ "1": {
4
+ "BLOCK_SIZE_M": 16,
5
+ "BLOCK_SIZE_N": 128,
6
+ "BLOCK_SIZE_K": 256,
7
+ "GROUP_SIZE_M": 1,
8
+ "num_warps": 4,
9
+ "num_stages": 2,
10
+ "waves_per_eu": 1
11
+ },
12
+ "2": {
13
+ "BLOCK_SIZE_M": 16,
14
+ "BLOCK_SIZE_N": 256,
15
+ "BLOCK_SIZE_K": 128,
16
+ "GROUP_SIZE_M": 1,
17
+ "num_warps": 8,
18
+ "num_stages": 2,
19
+ "waves_per_eu": 2
20
+ },
21
+ "4": {
22
+ "BLOCK_SIZE_M": 16,
23
+ "BLOCK_SIZE_N": 128,
24
+ "BLOCK_SIZE_K": 256,
25
+ "GROUP_SIZE_M": 1,
26
+ "num_warps": 8,
27
+ "num_stages": 2,
28
+ "waves_per_eu": 1
29
+ },
30
+ "8": {
31
+ "BLOCK_SIZE_M": 16,
32
+ "BLOCK_SIZE_N": 128,
33
+ "BLOCK_SIZE_K": 128,
34
+ "GROUP_SIZE_M": 4,
35
+ "num_warps": 8,
36
+ "num_stages": 2,
37
+ "waves_per_eu": 0
38
+ },
39
+ "16": {
40
+ "BLOCK_SIZE_M": 16,
41
+ "BLOCK_SIZE_N": 128,
42
+ "BLOCK_SIZE_K": 128,
43
+ "GROUP_SIZE_M": 4,
44
+ "num_warps": 8,
45
+ "num_stages": 2,
46
+ "waves_per_eu": 0
47
+ },
48
+ "24": {
49
+ "BLOCK_SIZE_M": 16,
50
+ "BLOCK_SIZE_N": 128,
51
+ "BLOCK_SIZE_K": 256,
52
+ "GROUP_SIZE_M": 4,
53
+ "num_warps": 8,
54
+ "num_stages": 2,
55
+ "waves_per_eu": 4
56
+ },
57
+ "32": {
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 128,
60
+ "BLOCK_SIZE_K": 256,
61
+ "GROUP_SIZE_M": 8,
62
+ "num_warps": 8,
63
+ "num_stages": 2,
64
+ "waves_per_eu": 4
65
+ },
66
+ "48": {
67
+ "BLOCK_SIZE_M": 16,
68
+ "BLOCK_SIZE_N": 128,
69
+ "BLOCK_SIZE_K": 128,
70
+ "GROUP_SIZE_M": 4,
71
+ "num_warps": 8,
72
+ "num_stages": 2,
73
+ "waves_per_eu": 1
74
+ },
75
+ "64": {
76
+ "BLOCK_SIZE_M": 16,
77
+ "BLOCK_SIZE_N": 128,
78
+ "BLOCK_SIZE_K": 128,
79
+ "GROUP_SIZE_M": 16,
80
+ "num_warps": 8,
81
+ "num_stages": 2,
82
+ "waves_per_eu": 2
83
+ },
84
+ "96": {
85
+ "BLOCK_SIZE_M": 16,
86
+ "BLOCK_SIZE_N": 128,
87
+ "BLOCK_SIZE_K": 128,
88
+ "GROUP_SIZE_M": 16,
89
+ "num_warps": 8,
90
+ "num_stages": 2,
91
+ "waves_per_eu": 2
92
+ },
93
+ "128": {
94
+ "BLOCK_SIZE_M": 16,
95
+ "BLOCK_SIZE_N": 128,
96
+ "BLOCK_SIZE_K": 128,
97
+ "GROUP_SIZE_M": 16,
98
+ "num_warps": 8,
99
+ "num_stages": 2,
100
+ "waves_per_eu": 4
101
+ },
102
+ "256": {
103
+ "BLOCK_SIZE_M": 32,
104
+ "BLOCK_SIZE_N": 128,
105
+ "BLOCK_SIZE_K": 128,
106
+ "GROUP_SIZE_M": 8,
107
+ "num_warps": 8,
108
+ "num_stages": 2,
109
+ "waves_per_eu": 2
110
+ },
111
+ "512": {
112
+ "BLOCK_SIZE_M": 16,
113
+ "BLOCK_SIZE_N": 128,
114
+ "BLOCK_SIZE_K": 256,
115
+ "GROUP_SIZE_M": 8,
116
+ "num_warps": 4,
117
+ "num_stages": 2,
118
+ "waves_per_eu": 4
119
+ },
120
+ "1024": {
121
+ "BLOCK_SIZE_M": 32,
122
+ "BLOCK_SIZE_N": 128,
123
+ "BLOCK_SIZE_K": 256,
124
+ "GROUP_SIZE_M": 4,
125
+ "num_warps": 4,
126
+ "num_stages": 2,
127
+ "waves_per_eu": 2
128
+ },
129
+ "1536": {
130
+ "BLOCK_SIZE_M": 32,
131
+ "BLOCK_SIZE_N": 128,
132
+ "BLOCK_SIZE_K": 256,
133
+ "GROUP_SIZE_M": 8,
134
+ "num_warps": 4,
135
+ "num_stages": 2,
136
+ "waves_per_eu": 2
137
+ },
138
+ "2048": {
139
+ "BLOCK_SIZE_M": 32,
140
+ "BLOCK_SIZE_N": 128,
141
+ "BLOCK_SIZE_K": 128,
142
+ "GROUP_SIZE_M": 4,
143
+ "num_warps": 4,
144
+ "num_stages": 2,
145
+ "waves_per_eu": 0
146
+ },
147
+ "3072": {
148
+ "BLOCK_SIZE_M": 32,
149
+ "BLOCK_SIZE_N": 128,
150
+ "BLOCK_SIZE_K": 256,
151
+ "GROUP_SIZE_M": 8,
152
+ "num_warps": 4,
153
+ "num_stages": 2,
154
+ "waves_per_eu": 0
155
+ },
156
+ "4096": {
157
+ "BLOCK_SIZE_M": 32,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 128,
160
+ "GROUP_SIZE_M": 8,
161
+ "num_warps": 4,
162
+ "num_stages": 2,
163
+ "waves_per_eu": 0
164
+ }
165
+}
vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=AMD_Radeon_R9700,dtype=fp8_w8a8,block_shape=[128,128].json
new
+165
@@ -0,0 +1,165 @@
1
+{
2
+ "triton_version": "3.6.0",
3
+ "1": {
4
+ "BLOCK_SIZE_M": 16,
5
+ "BLOCK_SIZE_N": 128,
6
+ "BLOCK_SIZE_K": 256,
7
+ "GROUP_SIZE_M": 1,
8
+ "num_warps": 4,
9
+ "num_stages": 2,
10
+ "waves_per_eu": 4
11
+ },
12
+ "2": {
13
+ "BLOCK_SIZE_M": 16,
14
+ "BLOCK_SIZE_N": 128,
15
+ "BLOCK_SIZE_K": 256,
16
+ "GROUP_SIZE_M": 1,
17
+ "num_warps": 4,
18
+ "num_stages": 2,
19
+ "waves_per_eu": 4
20
+ },
21
+ "4": {
22
+ "BLOCK_SIZE_M": 16,
23
+ "BLOCK_SIZE_N": 128,
24
+ "BLOCK_SIZE_K": 256,
25
+ "GROUP_SIZE_M": 1,
26
+ "num_warps": 8,
27
+ "num_stages": 2,
28
+ "waves_per_eu": 1
29
+ },
30
+ "8": {
31
+ "BLOCK_SIZE_M": 16,
32
+ "BLOCK_SIZE_N": 128,
33
+ "BLOCK_SIZE_K": 128,
34
+ "GROUP_SIZE_M": 4,
35
+ "num_warps": 8,
36
+ "num_stages": 2,
37
+ "waves_per_eu": 0
38
+ },
39
+ "16": {
40
+ "BLOCK_SIZE_M": 16,
41
+ "BLOCK_SIZE_N": 128,
42
+ "BLOCK_SIZE_K": 256,
43
+ "GROUP_SIZE_M": 8,
44
+ "num_warps": 8,
45
+ "num_stages": 2,
46
+ "waves_per_eu": 0
47
+ },
48
+ "24": {
49
+ "BLOCK_SIZE_M": 16,
50
+ "BLOCK_SIZE_N": 128,
51
+ "BLOCK_SIZE_K": 128,
52
+ "GROUP_SIZE_M": 4,
53
+ "num_warps": 8,
54
+ "num_stages": 2,
55
+ "waves_per_eu": 4
56
+ },
57
+ "32": {
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 128,
60
+ "BLOCK_SIZE_K": 128,
61
+ "GROUP_SIZE_M": 8,
62
+ "num_warps": 8,
63
+ "num_stages": 2,
64
+ "waves_per_eu": 4
65
+ },
66
+ "48": {
67
+ "BLOCK_SIZE_M": 16,
68
+ "BLOCK_SIZE_N": 128,
69
+ "BLOCK_SIZE_K": 128,
70
+ "GROUP_SIZE_M": 8,
71
+ "num_warps": 8,
72
+ "num_stages": 2,
73
+ "waves_per_eu": 1
74
+ },
75
+ "64": {
76
+ "BLOCK_SIZE_M": 16,
77
+ "BLOCK_SIZE_N": 128,
78
+ "BLOCK_SIZE_K": 256,
79
+ "GROUP_SIZE_M": 16,
80
+ "num_warps": 8,
81
+ "num_stages": 2,
82
+ "waves_per_eu": 2
83
+ },
84
+ "96": {
85
+ "BLOCK_SIZE_M": 32,
86
+ "BLOCK_SIZE_N": 128,
87
+ "BLOCK_SIZE_K": 256,
88
+ "GROUP_SIZE_M": 16,
89
+ "num_warps": 8,
90
+ "num_stages": 2,
91
+ "waves_per_eu": 4
92
+ },
93
+ "128": {
94
+ "BLOCK_SIZE_M": 32,
95
+ "BLOCK_SIZE_N": 128,
96
+ "BLOCK_SIZE_K": 128,
97
+ "GROUP_SIZE_M": 8,
98
+ "num_warps": 8,
99
+ "num_stages": 2,
100
+ "waves_per_eu": 2
101
+ },
102
+ "256": {
103
+ "BLOCK_SIZE_M": 64,
104
+ "BLOCK_SIZE_N": 128,
105
+ "BLOCK_SIZE_K": 128,
106
+ "GROUP_SIZE_M": 4,
107
+ "num_warps": 8,
108
+ "num_stages": 2,
109
+ "waves_per_eu": 2
110
+ },
111
+ "512": {
112
+ "BLOCK_SIZE_M": 32,
113
+ "BLOCK_SIZE_N": 128,
114
+ "BLOCK_SIZE_K": 256,
115
+ "GROUP_SIZE_M": 4,
116
+ "num_warps": 4,
117
+ "num_stages": 2,
118
+ "waves_per_eu": 2
119
+ },
120
+ "1024": {
121
+ "BLOCK_SIZE_M": 32,
122
+ "BLOCK_SIZE_N": 128,
123
+ "BLOCK_SIZE_K": 128,
124
+ "GROUP_SIZE_M": 4,
125
+ "num_warps": 4,
126
+ "num_stages": 2,
127
+ "waves_per_eu": 1
128
+ },
129
+ "1536": {
130
+ "BLOCK_SIZE_M": 32,
131
+ "BLOCK_SIZE_N": 128,
132
+ "BLOCK_SIZE_K": 256,
133
+ "GROUP_SIZE_M": 8,
134
+ "num_warps": 4,
135
+ "num_stages": 2,
136
+ "waves_per_eu": 0
137
+ },
138
+ "2048": {
139
+ "BLOCK_SIZE_M": 32,
140
+ "BLOCK_SIZE_N": 256,
141
+ "BLOCK_SIZE_K": 128,
142
+ "GROUP_SIZE_M": 8,
143
+ "num_warps": 8,
144
+ "num_stages": 2,
145
+ "waves_per_eu": 1
146
+ },
147
+ "3072": {
148
+ "BLOCK_SIZE_M": 32,
149
+ "BLOCK_SIZE_N": 128,
150
+ "BLOCK_SIZE_K": 128,
151
+ "GROUP_SIZE_M": 4,
152
+ "num_warps": 4,
153
+ "num_stages": 2,
154
+ "waves_per_eu": 0
155
+ },
156
+ "4096": {
157
+ "BLOCK_SIZE_M": 32,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 256,
160
+ "GROUP_SIZE_M": 4,
161
+ "num_warps": 4,
162
+ "num_stages": 2,
163
+ "waves_per_eu": 0
164
+ }
165
+}
vllm/model_executor/layers/fused_moe/fused_moe.py
+4
-1
@@ -1923,14 +1923,17 @@ class TritonExperts(mk.FusedMoEExpertsModular):
1923
) -> bool:
1924
p = current_platform
1925
if p.is_rocm():
1926
- from vllm.platforms.rocm import on_gfx9
1926
+ from vllm.platforms.rocm import on_gfx9, on_gfx12x
1927
1928
is_rocm_on_gfx9 = on_gfx9()
1929
+ is_rocm_on_gfx12x = on_gfx12x()
1930
else:
1931
is_rocm_on_gfx9 = False
1932
+ is_rocm_on_gfx12x = False
1933
1934
device_supports_fp8 = (
1935
is_rocm_on_gfx9
1936
+ or is_rocm_on_gfx12x
1937
or (p.is_cuda() and p.has_device_capability((8, 9)))
1938
or p.is_xpu()
1939
)
vllm/platforms/rocm.py
+9
-3
@@ -64,6 +64,7 @@ _ROCM_DEVICE_ID_NAME_MAP: dict[str, str] = {
64
"0x74a9": "AMD_Instinct_MI300X_HF",
65
"0x74bd": "AMD_Instinct_MI300X_HF",
66
"0x744c": "AMD_Radeon_RX7900XTX",
67
+ "0x7551": "AMD_Radeon_R9700",
68
}
69
70
@@ -178,6 +179,7 @@ def _get_gcn_arch() -> str:
179
_GCN_ARCH = _get_gcn_arch()
180
181
_ON_GFX1X = any(arch in _GCN_ARCH for arch in ["gfx11", "gfx12"])
182
+_ON_GFX12X = any(arch in _GCN_ARCH for arch in ["gfx12"])
183
_ON_MI3XX = any(arch in _GCN_ARCH for arch in ["gfx942", "gfx950"])
184
_ON_GFX9 = any(arch in _GCN_ARCH for arch in ["gfx90a", "gfx942", "gfx950"])
185
_ON_GFX942 = "gfx942" in _GCN_ARCH
@@ -259,6 +261,10 @@ def on_gfx1x() -> bool:
261
return _ON_GFX1X
262
263
264
+def on_gfx12x() -> bool:
265
+ return _ON_GFX12X
266
+
267
+
268
def on_mi3xx() -> bool:
269
return _ON_MI3XX
270
@@ -636,9 +642,9 @@ class RocmPlatform(Platform):
642
physical_device_id = cls.device_id_to_physical_device_id(device_id)
643
handle = amdsmi_get_processor_handles()[physical_device_id]
644
asic_info = amdsmi_get_gpu_asic_info(handle)
639
- device_name: str = asic_info["device_id"]
640
- if device_name in _ROCM_DEVICE_ID_NAME_MAP:
641
- return _ROCM_DEVICE_ID_NAME_MAP[device_name]
645
+ asic_info_device_id: str = asic_info["device_id"]
646
+ if asic_info_device_id in _ROCM_DEVICE_ID_NAME_MAP:
647
+ return _ROCM_DEVICE_ID_NAME_MAP[asic_info_device_id]
648
return asic_info["market_name"]
649
650
@classmethod