A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] Enable FP8 QKV in MoE and refine kernel tuning script (#5039)
Cody Yu committed
May 31, 2024 at 14:29 UTC
e9899fb7a4d9e032198d26ef84f1dd2cfd9621aa
8 files changed
+711
-114
benchmarks/kernels/benchmark_mixtral_moe.py
+36
-12
@@ -11,25 +11,36 @@ from tqdm import tqdm
11
from vllm.model_executor.layers.fused_moe import (fused_moe,
12
get_config_file_name)
13
14
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
14
16
-
17
-def main(dtype: str):
15
+def main(model, tp_size, gpu, dtype: str):
16
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
17
method = fused_moe
18
for bs in [
19
1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
20
2048, 3072, 4096
21
]:
23
- run_grid(bs, method=method, dtype=dtype)
24
-
25
-
26
-def run_grid(bs, method, dtype: str):
27
- d_model = 4096
22
+ run_grid(bs,
23
+ model=model,
24
+ method=method,
25
+ gpu=gpu,
26
+ tp_size=tp_size,
27
+ dtype=dtype)
28
+
29
+
30
+def run_grid(bs, model, method, gpu, tp_size, dtype: str):
31
+ if model == '8x7B':
32
+ d_model = 4096
33
+ model_intermediate_size = 14336
34
+ num_layers = 32
35
+ elif model == '8x22B':
36
+ d_model = 6144
37
+ model_intermediate_size = 16384
38
+ num_layers = 56
39
+ else:
40
+ raise ValueError(f'Unsupported Mixtral model {model}')
41
num_total_experts = 8
42
top_k = 2
30
- tp_size = 2
31
- model_intermediate_size = 14336
32
- num_layers = 32
43
+ # tp_size = 2
44
num_calls = 100
45
46
num_warmup_trials = 1
@@ -211,5 +222,18 @@ if __name__ == "__main__":
222
choices=['float8', 'float16'],
223
help='Data type used for fused_moe kernel computations',
224
)
225
+ parser.add_argument('--model',
226
+ type=str,
227
+ default='8x7B',
228
+ choices=['8x7B', '8x22B'],
229
+ help='The Mixtral model to benchmark')
230
+ parser.add_argument('--tp-size',
231
+ type=int,
232
+ default=2,
233
+ help='Tensor paralleli size')
234
+ parser.add_argument('--gpu',
235
+ type=int,
236
+ default=0,
237
+ help="GPU ID for benchmarking")
238
args = parser.parse_args()
215
- sys.exit(main(args.dtype))
239
+ sys.exit(main(args.model, args.tp_size, args.gpu, args.dtype))
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
new
+138
@@ -0,0 +1,138 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 64,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 128,
6
+ "GROUP_SIZE_M": 64,
7
+ "num_warps": 8,
8
+ "num_stages": 5
9
+ },
10
+ "2": {
11
+ "BLOCK_SIZE_M": 64,
12
+ "BLOCK_SIZE_N": 128,
13
+ "BLOCK_SIZE_K": 256,
14
+ "GROUP_SIZE_M": 64,
15
+ "num_warps": 4,
16
+ "num_stages": 3
17
+ },
18
+ "4": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 128,
21
+ "BLOCK_SIZE_K": 256,
22
+ "GROUP_SIZE_M": 1,
23
+ "num_warps": 4,
24
+ "num_stages": 4
25
+ },
26
+ "8": {
27
+ "BLOCK_SIZE_M": 64,
28
+ "BLOCK_SIZE_N": 128,
29
+ "BLOCK_SIZE_K": 256,
30
+ "GROUP_SIZE_M": 32,
31
+ "num_warps": 4,
32
+ "num_stages": 4
33
+ },
34
+ "16": {
35
+ "BLOCK_SIZE_M": 64,
36
+ "BLOCK_SIZE_N": 64,
37
+ "BLOCK_SIZE_K": 128,
38
+ "GROUP_SIZE_M": 1,
39
+ "num_warps": 4,
40
+ "num_stages": 3
41
+ },
42
+ "24": {
43
+ "BLOCK_SIZE_M": 64,
44
+ "BLOCK_SIZE_N": 128,
45
+ "BLOCK_SIZE_K": 256,
46
+ "GROUP_SIZE_M": 1,
47
+ "num_warps": 4,
48
+ "num_stages": 4
49
+ },
50
+ "32": {
51
+ "BLOCK_SIZE_M": 64,
52
+ "BLOCK_SIZE_N": 128,
53
+ "BLOCK_SIZE_K": 256,
54
+ "GROUP_SIZE_M": 1,
55
+ "num_warps": 4,
56
+ "num_stages": 4
57
+ },
58
+ "48": {
59
+ "BLOCK_SIZE_M": 64,
60
+ "BLOCK_SIZE_N": 128,
61
+ "BLOCK_SIZE_K": 256,
62
+ "GROUP_SIZE_M": 1,
63
+ "num_warps": 4,
64
+ "num_stages": 4
65
+ },
66
+ "64": {
67
+ "BLOCK_SIZE_M": 64,
68
+ "BLOCK_SIZE_N": 128,
69
+ "BLOCK_SIZE_K": 256,
70
+ "GROUP_SIZE_M": 1,
71
+ "num_warps": 4,
72
+ "num_stages": 4
73
+ },
74
+ "96": {
75
+ "BLOCK_SIZE_M": 64,
76
+ "BLOCK_SIZE_N": 128,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 1,
79
+ "num_warps": 4,
80
+ "num_stages": 2
81
+ },
82
+ "128": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 128,
85
+ "BLOCK_SIZE_K": 256,
86
+ "GROUP_SIZE_M": 1,
87
+ "num_warps": 4,
88
+ "num_stages": 2
89
+ },
90
+ "256": {
91
+ "BLOCK_SIZE_M": 128,
92
+ "BLOCK_SIZE_N": 128,
93
+ "BLOCK_SIZE_K": 128,
94
+ "GROUP_SIZE_M": 1,
95
+ "num_warps": 8,
96
+ "num_stages": 3
97
+ },
98
+ "512": {
99
+ "BLOCK_SIZE_M": 128,
100
+ "BLOCK_SIZE_N": 256,
101
+ "BLOCK_SIZE_K": 128,
102
+ "GROUP_SIZE_M": 64,
103
+ "num_warps": 8,
104
+ "num_stages": 4
105
+ },
106
+ "1024": {
107
+ "BLOCK_SIZE_M": 128,
108
+ "BLOCK_SIZE_N": 256,
109
+ "BLOCK_SIZE_K": 128,
110
+ "GROUP_SIZE_M": 64,
111
+ "num_warps": 8,
112
+ "num_stages": 4
113
+ },
114
+ "1536": {
115
+ "BLOCK_SIZE_M": 128,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 128,
118
+ "GROUP_SIZE_M": 64,
119
+ "num_warps": 8,
120
+ "num_stages": 3
121
+ },
122
+ "2048": {
123
+ "BLOCK_SIZE_M": 128,
124
+ "BLOCK_SIZE_N": 256,
125
+ "BLOCK_SIZE_K": 128,
126
+ "GROUP_SIZE_M": 64,
127
+ "num_warps": 8,
128
+ "num_stages": 3
129
+ },
130
+ "3072": {
131
+ "BLOCK_SIZE_M": 128,
132
+ "BLOCK_SIZE_N": 256,
133
+ "BLOCK_SIZE_K": 128,
134
+ "GROUP_SIZE_M": 32,
135
+ "num_warps": 8,
136
+ "num_stages": 3
137
+ }
138
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
new
+146
@@ -0,0 +1,146 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 32,
5
+ "BLOCK_SIZE_K": 64,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 8,
8
+ "num_stages": 3
9
+ },
10
+ "2": {
11
+ "BLOCK_SIZE_M": 16,
12
+ "BLOCK_SIZE_N": 64,
13
+ "BLOCK_SIZE_K": 64,
14
+ "GROUP_SIZE_M": 16,
15
+ "num_warps": 4,
16
+ "num_stages": 5
17
+ },
18
+ "4": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 32,
21
+ "BLOCK_SIZE_K": 256,
22
+ "GROUP_SIZE_M": 64,
23
+ "num_warps": 8,
24
+ "num_stages": 5
25
+ },
26
+ "8": {
27
+ "BLOCK_SIZE_M": 16,
28
+ "BLOCK_SIZE_N": 64,
29
+ "BLOCK_SIZE_K": 256,
30
+ "GROUP_SIZE_M": 1,
31
+ "num_warps": 4,
32
+ "num_stages": 3
33
+ },
34
+ "16": {
35
+ "BLOCK_SIZE_M": 128,
36
+ "BLOCK_SIZE_N": 64,
37
+ "BLOCK_SIZE_K": 64,
38
+ "GROUP_SIZE_M": 1,
39
+ "num_warps": 8,
40
+ "num_stages": 3
41
+ },
42
+ "24": {
43
+ "BLOCK_SIZE_M": 64,
44
+ "BLOCK_SIZE_N": 256,
45
+ "BLOCK_SIZE_K": 128,
46
+ "GROUP_SIZE_M": 64,
47
+ "num_warps": 8,
48
+ "num_stages": 2
49
+ },
50
+ "32": {
51
+ "BLOCK_SIZE_M": 64,
52
+ "BLOCK_SIZE_N": 256,
53
+ "BLOCK_SIZE_K": 64,
54
+ "GROUP_SIZE_M": 1,
55
+ "num_warps": 8,
56
+ "num_stages": 3
57
+ },
58
+ "48": {
59
+ "BLOCK_SIZE_M": 64,
60
+ "BLOCK_SIZE_N": 256,
61
+ "BLOCK_SIZE_K": 128,
62
+ "GROUP_SIZE_M": 64,
63
+ "num_warps": 8,
64
+ "num_stages": 4
65
+ },
66
+ "64": {
67
+ "BLOCK_SIZE_M": 128,
68
+ "BLOCK_SIZE_N": 256,
69
+ "BLOCK_SIZE_K": 128,
70
+ "GROUP_SIZE_M": 64,
71
+ "num_warps": 8,
72
+ "num_stages": 2
73
+ },
74
+ "96": {
75
+ "BLOCK_SIZE_M": 64,
76
+ "BLOCK_SIZE_N": 256,
77
+ "BLOCK_SIZE_K": 128,
78
+ "GROUP_SIZE_M": 16,
79
+ "num_warps": 4,
80
+ "num_stages": 4
81
+ },
82
+ "128": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 256,
85
+ "BLOCK_SIZE_K": 64,
86
+ "GROUP_SIZE_M": 32,
87
+ "num_warps": 8,
88
+ "num_stages": 3
89
+ },
90
+ "256": {
91
+ "BLOCK_SIZE_M": 128,
92
+ "BLOCK_SIZE_N": 64,
93
+ "BLOCK_SIZE_K": 128,
94
+ "GROUP_SIZE_M": 64,
95
+ "num_warps": 8,
96
+ "num_stages": 4
97
+ },
98
+ "512": {
99
+ "BLOCK_SIZE_M": 64,
100
+ "BLOCK_SIZE_N": 256,
101
+ "BLOCK_SIZE_K": 128,
102
+ "GROUP_SIZE_M": 64,
103
+ "num_warps": 4,
104
+ "num_stages": 3
105
+ },
106
+ "1024": {
107
+ "BLOCK_SIZE_M": 128,
108
+ "BLOCK_SIZE_N": 256,
109
+ "BLOCK_SIZE_K": 128,
110
+ "GROUP_SIZE_M": 32,
111
+ "num_warps": 8,
112
+ "num_stages": 4
113
+ },
114
+ "1536": {
115
+ "BLOCK_SIZE_M": 128,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 128,
118
+ "GROUP_SIZE_M": 64,
119
+ "num_warps": 8,
120
+ "num_stages": 4
121
+ },
122
+ "2048": {
123
+ "BLOCK_SIZE_M": 128,
124
+ "BLOCK_SIZE_N": 256,
125
+ "BLOCK_SIZE_K": 128,
126
+ "GROUP_SIZE_M": 64,
127
+ "num_warps": 8,
128
+ "num_stages": 4
129
+ },
130
+ "3072": {
131
+ "BLOCK_SIZE_M": 128,
132
+ "BLOCK_SIZE_N": 256,
133
+ "BLOCK_SIZE_K": 128,
134
+ "GROUP_SIZE_M": 32,
135
+ "num_warps": 8,
136
+ "num_stages": 4
137
+ },
138
+ "4096": {
139
+ "BLOCK_SIZE_M": 128,
140
+ "BLOCK_SIZE_N": 256,
141
+ "BLOCK_SIZE_K": 128,
142
+ "GROUP_SIZE_M": 32,
143
+ "num_warps": 8,
144
+ "num_stages": 4
145
+ }
146
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
+57
-51
@@ -3,61 +3,59 @@
3
"BLOCK_SIZE_M": 16,
4
"BLOCK_SIZE_N": 32,
5
"BLOCK_SIZE_K": 64,
6
- "GROUP_SIZE_M": 1
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 8,
8
+ "num_stages": 4
9
},
10
"2": {
11
"BLOCK_SIZE_M": 16,
10
- "BLOCK_SIZE_N": 32,
12
+ "BLOCK_SIZE_N": 256,
13
"BLOCK_SIZE_K": 64,
12
- "GROUP_SIZE_M": 1
14
+ "GROUP_SIZE_M": 1,
15
+ "num_warps": 8,
16
+ "num_stages": 3
17
},
18
"4": {
15
- "BLOCK_SIZE_M": 16,
16
- "BLOCK_SIZE_N": 32,
17
- "BLOCK_SIZE_K": 64,
18
- "GROUP_SIZE_M": 1
19
+ "BLOCK_SIZE_M": 256,
20
+ "BLOCK_SIZE_N": 64,
21
+ "BLOCK_SIZE_K": 128,
22
+ "GROUP_SIZE_M": 1,
23
+ "num_warps": 4,
24
+ "num_stages": 2
25
},
26
"8": {
27
"BLOCK_SIZE_M": 64,
22
- "BLOCK_SIZE_N": 64,
23
- "BLOCK_SIZE_K": 256,
24
- "GROUP_SIZE_M": 1,
28
+ "BLOCK_SIZE_N": 256,
29
+ "BLOCK_SIZE_K": 64,
30
+ "GROUP_SIZE_M": 32,
31
"num_warps": 8,
26
- "num_stages": 5
32
+ "num_stages": 2
33
},
34
"16": {
29
- "BLOCK_SIZE_M": 64,
35
+ "BLOCK_SIZE_M": 256,
36
"BLOCK_SIZE_N": 128,
37
"BLOCK_SIZE_K": 64,
32
- "GROUP_SIZE_M": 16,
33
- "num_warps": 4,
34
- "num_stages": 5
35
- },
36
- "24": {
37
- "BLOCK_SIZE_M": 128,
38
- "BLOCK_SIZE_N": 64,
39
- "BLOCK_SIZE_K": 128,
38
"GROUP_SIZE_M": 1,
39
"num_warps": 8,
40
"num_stages": 5
41
},
44
- "32": {
42
+ "24": {
43
"BLOCK_SIZE_M": 64,
44
"BLOCK_SIZE_N": 256,
45
"BLOCK_SIZE_K": 128,
48
- "GROUP_SIZE_M": 16,
49
- "num_warps": 8,
46
+ "GROUP_SIZE_M": 64,
47
+ "num_warps": 4,
48
"num_stages": 4
49
},
52
- "48": {
50
+ "32": {
51
"BLOCK_SIZE_M": 64,
54
- "BLOCK_SIZE_N": 256,
55
- "BLOCK_SIZE_K": 128,
56
- "GROUP_SIZE_M": 64,
57
- "num_warps": 8,
58
- "num_stages": 3
52
+ "BLOCK_SIZE_N": 128,
53
+ "BLOCK_SIZE_K": 256,
54
+ "GROUP_SIZE_M": 1,
55
+ "num_warps": 4,
56
+ "num_stages": 4
57
},
60
- "64": {
58
+ "48": {
59
"BLOCK_SIZE_M": 64,
60
"BLOCK_SIZE_N": 256,
61
"BLOCK_SIZE_K": 64,
@@ -65,37 +63,45 @@
63
"num_warps": 4,
64
"num_stages": 4
65
},
68
- "96": {
69
- "BLOCK_SIZE_M": 128,
70
- "BLOCK_SIZE_N": 64,
66
+ "64": {
67
+ "BLOCK_SIZE_M": 64,
68
+ "BLOCK_SIZE_N": 128,
69
"BLOCK_SIZE_K": 256,
72
- "GROUP_SIZE_M": 32,
73
- "num_warps": 8,
74
- "num_stages": 2
75
- },
76
- "128": {
77
- "BLOCK_SIZE_M": 128,
78
- "BLOCK_SIZE_N": 64,
79
- "BLOCK_SIZE_K": 128,
80
- "GROUP_SIZE_M": 32,
70
+ "GROUP_SIZE_M": 1,
71
"num_warps": 4,
82
- "num_stages": 3
72
+ "num_stages": 2
73
},
84
- "256": {
74
+ "96": {
75
"BLOCK_SIZE_M": 128,
76
"BLOCK_SIZE_N": 32,
77
"BLOCK_SIZE_K": 128,
88
- "GROUP_SIZE_M": 16,
78
+ "GROUP_SIZE_M": 1,
79
"num_warps": 4,
80
"num_stages": 5
81
},
92
- "512": {
82
+ "128": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 64,
85
+ "BLOCK_SIZE_K": 256,
86
+ "GROUP_SIZE_M": 32,
87
+ "num_warps": 8,
88
+ "num_stages": 5
89
+ },
90
+ "256": {
91
"BLOCK_SIZE_M": 64,
92
"BLOCK_SIZE_N": 128,
93
"BLOCK_SIZE_K": 256,
96
- "GROUP_SIZE_M": 64,
97
- "num_warps": 4,
98
- "num_stages": 2
94
+ "GROUP_SIZE_M": 1,
95
+ "num_warps": 8,
96
+ "num_stages": 3
97
+ },
98
+ "512": {
99
+ "BLOCK_SIZE_M": 128,
100
+ "BLOCK_SIZE_N": 64,
101
+ "BLOCK_SIZE_K": 128,
102
+ "GROUP_SIZE_M": 16,
103
+ "num_warps": 8,
104
+ "num_stages": 3
105
},
106
"1024": {
107
"BLOCK_SIZE_M": 128,
@@ -109,7 +115,7 @@
115
"BLOCK_SIZE_M": 128,
116
"BLOCK_SIZE_N": 256,
117
"BLOCK_SIZE_K": 128,
112
- "GROUP_SIZE_M": 32,
118
+ "GROUP_SIZE_M": 64,
119
"num_warps": 8,
120
"num_stages": 4
121
},
@@ -125,7 +131,7 @@
131
"BLOCK_SIZE_M": 128,
132
"BLOCK_SIZE_N": 256,
133
"BLOCK_SIZE_K": 128,
128
- "GROUP_SIZE_M": 32,
134
+ "GROUP_SIZE_M": 16,
135
"num_warps": 8,
136
"num_stages": 4
137
},
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
new
+146
@@ -0,0 +1,146 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 32,
5
+ "BLOCK_SIZE_K": 64,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 5
9
+ },
10
+ "2": {
11
+ "BLOCK_SIZE_M": 64,
12
+ "BLOCK_SIZE_N": 64,
13
+ "BLOCK_SIZE_K": 64,
14
+ "GROUP_SIZE_M": 16,
15
+ "num_warps": 4,
16
+ "num_stages": 4
17
+ },
18
+ "4": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 128,
21
+ "BLOCK_SIZE_K": 128,
22
+ "GROUP_SIZE_M": 64,
23
+ "num_warps": 8,
24
+ "num_stages": 2
25
+ },
26
+ "8": {
27
+ "BLOCK_SIZE_M": 64,
28
+ "BLOCK_SIZE_N": 256,
29
+ "BLOCK_SIZE_K": 128,
30
+ "GROUP_SIZE_M": 32,
31
+ "num_warps": 4,
32
+ "num_stages": 3
33
+ },
34
+ "16": {
35
+ "BLOCK_SIZE_M": 64,
36
+ "BLOCK_SIZE_N": 128,
37
+ "BLOCK_SIZE_K": 256,
38
+ "GROUP_SIZE_M": 1,
39
+ "num_warps": 8,
40
+ "num_stages": 2
41
+ },
42
+ "24": {
43
+ "BLOCK_SIZE_M": 64,
44
+ "BLOCK_SIZE_N": 128,
45
+ "BLOCK_SIZE_K": 256,
46
+ "GROUP_SIZE_M": 1,
47
+ "num_warps": 4,
48
+ "num_stages": 3
49
+ },
50
+ "32": {
51
+ "BLOCK_SIZE_M": 64,
52
+ "BLOCK_SIZE_N": 64,
53
+ "BLOCK_SIZE_K": 128,
54
+ "GROUP_SIZE_M": 1,
55
+ "num_warps": 8,
56
+ "num_stages": 3
57
+ },
58
+ "48": {
59
+ "BLOCK_SIZE_M": 64,
60
+ "BLOCK_SIZE_N": 128,
61
+ "BLOCK_SIZE_K": 128,
62
+ "GROUP_SIZE_M": 64,
63
+ "num_warps": 4,
64
+ "num_stages": 2
65
+ },
66
+ "64": {
67
+ "BLOCK_SIZE_M": 64,
68
+ "BLOCK_SIZE_N": 64,
69
+ "BLOCK_SIZE_K": 256,
70
+ "GROUP_SIZE_M": 16,
71
+ "num_warps": 4,
72
+ "num_stages": 2
73
+ },
74
+ "96": {
75
+ "BLOCK_SIZE_M": 128,
76
+ "BLOCK_SIZE_N": 64,
77
+ "BLOCK_SIZE_K": 128,
78
+ "GROUP_SIZE_M": 1,
79
+ "num_warps": 4,
80
+ "num_stages": 3
81
+ },
82
+ "128": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 128,
85
+ "BLOCK_SIZE_K": 256,
86
+ "GROUP_SIZE_M": 1,
87
+ "num_warps": 8,
88
+ "num_stages": 2
89
+ },
90
+ "256": {
91
+ "BLOCK_SIZE_M": 128,
92
+ "BLOCK_SIZE_N": 128,
93
+ "BLOCK_SIZE_K": 128,
94
+ "GROUP_SIZE_M": 1,
95
+ "num_warps": 8,
96
+ "num_stages": 5
97
+ },
98
+ "512": {
99
+ "BLOCK_SIZE_M": 128,
100
+ "BLOCK_SIZE_N": 256,
101
+ "BLOCK_SIZE_K": 128,
102
+ "GROUP_SIZE_M": 64,
103
+ "num_warps": 8,
104
+ "num_stages": 4
105
+ },
106
+ "1024": {
107
+ "BLOCK_SIZE_M": 128,
108
+ "BLOCK_SIZE_N": 256,
109
+ "BLOCK_SIZE_K": 128,
110
+ "GROUP_SIZE_M": 64,
111
+ "num_warps": 8,
112
+ "num_stages": 4
113
+ },
114
+ "1536": {
115
+ "BLOCK_SIZE_M": 128,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 128,
118
+ "GROUP_SIZE_M": 64,
119
+ "num_warps": 8,
120
+ "num_stages": 4
121
+ },
122
+ "2048": {
123
+ "BLOCK_SIZE_M": 128,
124
+ "BLOCK_SIZE_N": 256,
125
+ "BLOCK_SIZE_K": 128,
126
+ "GROUP_SIZE_M": 64,
127
+ "num_warps": 8,
128
+ "num_stages": 4
129
+ },
130
+ "3072": {
131
+ "BLOCK_SIZE_M": 128,
132
+ "BLOCK_SIZE_N": 256,
133
+ "BLOCK_SIZE_K": 128,
134
+ "GROUP_SIZE_M": 32,
135
+ "num_warps": 8,
136
+ "num_stages": 4
137
+ },
138
+ "4096": {
139
+ "BLOCK_SIZE_M": 128,
140
+ "BLOCK_SIZE_N": 256,
141
+ "BLOCK_SIZE_K": 128,
142
+ "GROUP_SIZE_M": 16,
143
+ "num_warps": 8,
144
+ "num_stages": 4
145
+ }
146
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
+42
-42
@@ -2,104 +2,104 @@
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
"BLOCK_SIZE_N": 32,
5
- "BLOCK_SIZE_K": 128,
5
+ "BLOCK_SIZE_K": 64,
6
"GROUP_SIZE_M": 1,
7
"num_warps": 4,
8
- "num_stages": 4
8
+ "num_stages": 5
9
},
10
"2": {
11
- "BLOCK_SIZE_M": 128,
11
+ "BLOCK_SIZE_M": 64,
12
"BLOCK_SIZE_N": 64,
13
"BLOCK_SIZE_K": 128,
14
- "GROUP_SIZE_M": 1,
15
- "num_warps": 4,
14
+ "GROUP_SIZE_M": 32,
15
+ "num_warps": 8,
16
"num_stages": 4
17
},
18
"4": {
19
"BLOCK_SIZE_M": 64,
20
- "BLOCK_SIZE_N": 64,
21
- "BLOCK_SIZE_K": 64,
22
- "GROUP_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 128,
21
+ "BLOCK_SIZE_K": 256,
22
+ "GROUP_SIZE_M": 1,
23
"num_warps": 4,
24
- "num_stages": 4
24
+ "num_stages": 2
25
},
26
"8": {
27
- "BLOCK_SIZE_M": 64,
27
+ "BLOCK_SIZE_M": 128,
28
"BLOCK_SIZE_N": 128,
29
- "BLOCK_SIZE_K": 256,
30
- "GROUP_SIZE_M": 64,
31
- "num_warps": 8,
32
- "num_stages": 4
29
+ "BLOCK_SIZE_K": 128,
30
+ "GROUP_SIZE_M": 1,
31
+ "num_warps": 4,
32
+ "num_stages": 3
33
},
34
"16": {
35
"BLOCK_SIZE_M": 64,
36
- "BLOCK_SIZE_N": 256,
37
- "BLOCK_SIZE_K": 128,
36
+ "BLOCK_SIZE_N": 128,
37
+ "BLOCK_SIZE_K": 256,
38
"GROUP_SIZE_M": 1,
39
- "num_warps": 8,
39
+ "num_warps": 4,
40
"num_stages": 4
41
},
42
"24": {
43
"BLOCK_SIZE_M": 64,
44
- "BLOCK_SIZE_N": 256,
44
+ "BLOCK_SIZE_N": 128,
45
"BLOCK_SIZE_K": 128,
46
"GROUP_SIZE_M": 1,
47
- "num_warps": 8,
48
- "num_stages": 4
47
+ "num_warps": 4,
48
+ "num_stages": 5
49
},
50
"32": {
51
"BLOCK_SIZE_M": 64,
52
"BLOCK_SIZE_N": 128,
53
- "BLOCK_SIZE_K": 128,
54
- "GROUP_SIZE_M": 16,
55
- "num_warps": 8,
53
+ "BLOCK_SIZE_K": 256,
54
+ "GROUP_SIZE_M": 64,
55
+ "num_warps": 4,
56
"num_stages": 4
57
},
58
"48": {
59
"BLOCK_SIZE_M": 64,
60
- "BLOCK_SIZE_N": 128,
61
- "BLOCK_SIZE_K": 128,
62
- "GROUP_SIZE_M": 32,
60
+ "BLOCK_SIZE_N": 64,
61
+ "BLOCK_SIZE_K": 256,
62
+ "GROUP_SIZE_M": 1,
63
"num_warps": 4,
64
- "num_stages": 4
64
+ "num_stages": 3
65
},
66
"64": {
67
"BLOCK_SIZE_M": 64,
68
"BLOCK_SIZE_N": 128,
69
- "BLOCK_SIZE_K": 128,
70
- "GROUP_SIZE_M": 16,
71
- "num_warps": 8,
69
+ "BLOCK_SIZE_K": 256,
70
+ "GROUP_SIZE_M": 1,
71
+ "num_warps": 4,
72
"num_stages": 4
73
},
74
"96": {
75
"BLOCK_SIZE_M": 64,
76
- "BLOCK_SIZE_N": 32,
76
+ "BLOCK_SIZE_N": 128,
77
"BLOCK_SIZE_K": 256,
78
- "GROUP_SIZE_M": 32,
78
+ "GROUP_SIZE_M": 1,
79
"num_warps": 4,
80
"num_stages": 4
81
},
82
"128": {
83
"BLOCK_SIZE_M": 64,
84
- "BLOCK_SIZE_N": 128,
85
- "BLOCK_SIZE_K": 128,
86
- "GROUP_SIZE_M": 16,
87
- "num_warps": 8,
88
- "num_stages": 4
84
+ "BLOCK_SIZE_N": 64,
85
+ "BLOCK_SIZE_K": 256,
86
+ "GROUP_SIZE_M": 1,
87
+ "num_warps": 4,
88
+ "num_stages": 3
89
},
90
"256": {
91
"BLOCK_SIZE_M": 128,
92
- "BLOCK_SIZE_N": 256,
92
+ "BLOCK_SIZE_N": 128,
93
"BLOCK_SIZE_K": 128,
94
"GROUP_SIZE_M": 1,
95
"num_warps": 8,
96
- "num_stages": 4
96
+ "num_stages": 5
97
},
98
"512": {
99
"BLOCK_SIZE_M": 128,
100
"BLOCK_SIZE_N": 256,
101
"BLOCK_SIZE_K": 128,
102
- "GROUP_SIZE_M": 16,
102
+ "GROUP_SIZE_M": 64,
103
"num_warps": 8,
104
"num_stages": 4
105
},
@@ -115,7 +115,7 @@
115
"BLOCK_SIZE_M": 128,
116
"BLOCK_SIZE_N": 256,
117
"BLOCK_SIZE_K": 128,
118
- "GROUP_SIZE_M": 32,
118
+ "GROUP_SIZE_M": 64,
119
"num_warps": 8,
120
"num_stages": 4
121
},
@@ -139,7 +139,7 @@
139
"BLOCK_SIZE_M": 128,
140
"BLOCK_SIZE_N": 256,
141
"BLOCK_SIZE_K": 128,
142
- "GROUP_SIZE_M": 64,
142
+ "GROUP_SIZE_M": 16,
143
"num_warps": 8,
144
"num_stages": 4
145
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
new
+146
@@ -0,0 +1,146 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 64,
4
+ "BLOCK_SIZE_N": 128,
5
+ "BLOCK_SIZE_K": 64,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 8,
8
+ "num_stages": 5
9
+ },
10
+ "2": {
11
+ "BLOCK_SIZE_M": 128,
12
+ "BLOCK_SIZE_N": 64,
13
+ "BLOCK_SIZE_K": 256,
14
+ "GROUP_SIZE_M": 16,
15
+ "num_warps": 8,
16
+ "num_stages": 3
17
+ },
18
+ "4": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 128,
21
+ "BLOCK_SIZE_K": 256,
22
+ "GROUP_SIZE_M": 32,
23
+ "num_warps": 4,
24
+ "num_stages": 4
25
+ },
26
+ "8": {
27
+ "BLOCK_SIZE_M": 64,
28
+ "BLOCK_SIZE_N": 64,
29
+ "BLOCK_SIZE_K": 128,
30
+ "GROUP_SIZE_M": 1,
31
+ "num_warps": 4,
32
+ "num_stages": 5
33
+ },
34
+ "16": {
35
+ "BLOCK_SIZE_M": 64,
36
+ "BLOCK_SIZE_N": 128,
37
+ "BLOCK_SIZE_K": 256,
38
+ "GROUP_SIZE_M": 1,
39
+ "num_warps": 4,
40
+ "num_stages": 4
41
+ },
42
+ "24": {
43
+ "BLOCK_SIZE_M": 64,
44
+ "BLOCK_SIZE_N": 128,
45
+ "BLOCK_SIZE_K": 256,
46
+ "GROUP_SIZE_M": 1,
47
+ "num_warps": 4,
48
+ "num_stages": 4
49
+ },
50
+ "32": {
51
+ "BLOCK_SIZE_M": 64,
52
+ "BLOCK_SIZE_N": 128,
53
+ "BLOCK_SIZE_K": 256,
54
+ "GROUP_SIZE_M": 1,
55
+ "num_warps": 4,
56
+ "num_stages": 4
57
+ },
58
+ "48": {
59
+ "BLOCK_SIZE_M": 64,
60
+ "BLOCK_SIZE_N": 128,
61
+ "BLOCK_SIZE_K": 256,
62
+ "GROUP_SIZE_M": 1,
63
+ "num_warps": 4,
64
+ "num_stages": 4
65
+ },
66
+ "64": {
67
+ "BLOCK_SIZE_M": 64,
68
+ "BLOCK_SIZE_N": 128,
69
+ "BLOCK_SIZE_K": 256,
70
+ "GROUP_SIZE_M": 1,
71
+ "num_warps": 4,
72
+ "num_stages": 4
73
+ },
74
+ "96": {
75
+ "BLOCK_SIZE_M": 64,
76
+ "BLOCK_SIZE_N": 128,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 1,
79
+ "num_warps": 4,
80
+ "num_stages": 4
81
+ },
82
+ "128": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 128,
85
+ "BLOCK_SIZE_K": 256,
86
+ "GROUP_SIZE_M": 1,
87
+ "num_warps": 4,
88
+ "num_stages": 4
89
+ },
90
+ "256": {
91
+ "BLOCK_SIZE_M": 128,
92
+ "BLOCK_SIZE_N": 128,
93
+ "BLOCK_SIZE_K": 128,
94
+ "GROUP_SIZE_M": 1,
95
+ "num_warps": 8,
96
+ "num_stages": 5
97
+ },
98
+ "512": {
99
+ "BLOCK_SIZE_M": 128,
100
+ "BLOCK_SIZE_N": 256,
101
+ "BLOCK_SIZE_K": 128,
102
+ "GROUP_SIZE_M": 32,
103
+ "num_warps": 8,
104
+ "num_stages": 4
105
+ },
106
+ "1024": {
107
+ "BLOCK_SIZE_M": 128,
108
+ "BLOCK_SIZE_N": 256,
109
+ "BLOCK_SIZE_K": 128,
110
+ "GROUP_SIZE_M": 64,
111
+ "num_warps": 8,
112
+ "num_stages": 4
113
+ },
114
+ "1536": {
115
+ "BLOCK_SIZE_M": 128,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 128,
118
+ "GROUP_SIZE_M": 32,
119
+ "num_warps": 8,
120
+ "num_stages": 4
121
+ },
122
+ "2048": {
123
+ "BLOCK_SIZE_M": 128,
124
+ "BLOCK_SIZE_N": 256,
125
+ "BLOCK_SIZE_K": 128,
126
+ "GROUP_SIZE_M": 32,
127
+ "num_warps": 8,
128
+ "num_stages": 4
129
+ },
130
+ "3072": {
131
+ "BLOCK_SIZE_M": 128,
132
+ "BLOCK_SIZE_N": 256,
133
+ "BLOCK_SIZE_K": 128,
134
+ "GROUP_SIZE_M": 32,
135
+ "num_warps": 8,
136
+ "num_stages": 3
137
+ },
138
+ "4096": {
139
+ "BLOCK_SIZE_M": 128,
140
+ "BLOCK_SIZE_N": 256,
141
+ "BLOCK_SIZE_K": 128,
142
+ "GROUP_SIZE_M": 32,
143
+ "num_warps": 8,
144
+ "num_stages": 4
145
+ }
146
+}
vllm/model_executor/models/mixtral.py
-9
@@ -278,15 +278,6 @@ class MixtralAttention(nn.Module):
278
self.scaling = self.head_dim**-0.5
279
self.rope_theta = rope_theta
280
281
- if isinstance(
282
- quant_config,
283
- Fp8Config) and not quant_config.is_checkpoint_fp8_serialized:
284
- print_warning_once(
285
- "For Mixtral FP8 quantization, we currently do not quantize "
286
- "the attention layers until their FP8 performance is improved."
287
- )
288
- quant_config = None
289
-
281
self.qkv_proj = QKVParallelLinear(
282
hidden_size,
283
self.head_dim,