1
+import argparse
2
import json
3
import os
4
import sys
6
import torch
7
import torch.nn.functional as F
8
import triton
9
+from tqdm import tqdm
10
11
from vllm.model_executor.layers.fused_moe import (fused_moe,
12
get_config_file_name)
14
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
15
16
15
-def main():
17
+def main(dtype: str):
18
method = fused_moe
19
for bs in [
20
1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
21
2048, 3072, 4096
22
]:
21
- run_grid(bs, method=method)
23
+ run_grid(bs, method=method, dtype=dtype)
24
25
24
-def run_grid(bs, method):
26
+def run_grid(bs, method, dtype: str):
27
d_model = 4096
28
num_total_experts = 8
29
top_k = 2
36
num_trials = 1
37
38
configs = []
37
- if bs <= 16:
38
- BLOCK_SIZES_M = [16]
39
- elif bs <= 32:
40
- BLOCK_SIZES_M = [16, 32]
41
- elif bs <= 64:
42
- BLOCK_SIZES_M = [16, 32, 64]
43
- elif bs <= 128:
44
- BLOCK_SIZES_M = [16, 32, 64, 128]
45
- else:
46
- BLOCK_SIZES_M = [16, 32, 64, 128, 256]
39
40
for block_size_n in [32, 64, 128, 256]:
49
- for block_size_m in BLOCK_SIZES_M:
41
+ for block_size_m in [16, 32, 64, 128, 256]:
42
for block_size_k in [64, 128, 256]:
43
for group_size_m in [1, 16, 32, 64]:
44
for num_warps in [4, 8]:
53
- configs.append({
54
- "BLOCK_SIZE_M": block_size_m,
55
- "BLOCK_SIZE_N": block_size_n,
56
- "BLOCK_SIZE_K": block_size_k,
57
- "GROUP_SIZE_M": group_size_m,
58
- "num_warps": num_warps,
59
- "num_stages": 4,
60
- })
45
+ for num_stages in [2, 3, 4, 5]:
46
+ configs.append({
47
+ "BLOCK_SIZE_M": block_size_m,
48
+ "BLOCK_SIZE_N": block_size_n,
49
+ "BLOCK_SIZE_K": block_size_k,
50
+ "GROUP_SIZE_M": group_size_m,
51
+ "num_warps": num_warps,
52
+ "num_stages": num_stages,
53
+ })
54
55
best_config = None
56
best_time_us = 1e20
57
65
- for config in configs:
66
- print(f'{tp_size=} {bs=}')
67
- print(f'{config}')
58
+ print(f'{tp_size=} {bs=}')
59
+
60
+ for config in tqdm(configs):
61
# warmup
69
- print('warming up')
62
try:
63
for _ in range(num_warmup_trials):
64
run_timing(
71
model_intermediate_size=model_intermediate_size,
72
method=method,
73
config=config,
74
+ dtype=dtype,
75
)
76
except triton.runtime.autotuner.OutOfResources:
77
continue
78
79
# trial
87
- print('benchmarking')
80
for _ in range(num_trials):
81
kernel_dur_ms = run_timing(
82
num_calls=num_calls,
88
model_intermediate_size=model_intermediate_size,
89
method=method,
90
config=config,
91
+ dtype=dtype,
92
)
93
94
kernel_dur_us = 1000 * kernel_dur_ms
98
best_config = config
99
best_time_us = kernel_dur_us
100
108
- print(f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
109
- f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
110
- f'{d_model=} {model_intermediate_size=} {num_layers=}')
101
+ tqdm.write(
102
+ f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
103
+ f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
104
+ f'{d_model=} {model_intermediate_size=} {num_layers=}')
105
106
print("best_time_us", best_time_us)
107
print("best_config", best_config)
108
109
# holds Dict[str, Dict[str, int]]
110
filename = get_config_file_name(num_total_experts,
117
- model_intermediate_size // tp_size)
111
+ model_intermediate_size // tp_size,
112
+ "float8" if dtype == "float8" else None)
113
print(f"writing config to file {filename}")
114
existing_content = {}
115
if os.path.exists(filename):
123
124
def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
125
top_k: int, tp_size: int, model_intermediate_size: int, method,
131
- config) -> float:
126
+ config, dtype: str) -> float:
127
shard_intermediate_size = model_intermediate_size // tp_size
128
129
hidden_states = torch.rand(
130
(bs, d_model),
131
device="cuda:0",
137
- dtype=torch.bfloat16,
132
+ dtype=torch.float16,
133
)
134
140
- ws = torch.rand(
135
+ w1 = torch.rand(
136
(num_total_experts, 2 * shard_intermediate_size, d_model),
137
device=hidden_states.device,
138
dtype=hidden_states.dtype,
139
)
140
146
- w2s = torch.rand(
141
+ w2 = torch.rand(
142
(num_total_experts, d_model, shard_intermediate_size),
143
device=hidden_states.device,
144
dtype=hidden_states.dtype,
145
)
146
147
+ w1_scale = None
148
+ w2_scale = None
149
+ a1_scale = None
150
+ a2_scale = None
151
+
152
+ if dtype == "float8":
153
+ w1 = w1.to(torch.float8_e4m3fn)
154
+ w2 = w2.to(torch.float8_e4m3fn)
155
+ w1_scale = torch.ones(num_total_experts,
156
+ device=hidden_states.device,
157
+ dtype=torch.float32)
158
+ w2_scale = torch.ones(num_total_experts,
159
+ device=hidden_states.device,
160
+ dtype=torch.float32)
161
+ a1_scale = torch.ones(1,
162
+ device=hidden_states.device,
163
+ dtype=torch.float32)
164
+ a2_scale = torch.ones(1,
165
+ device=hidden_states.device,
166
+ dtype=torch.float32)
167
+
168
gating_output = F.softmax(torch.rand(
169
(num_calls, bs, num_total_experts),
170
device=hidden_states.device,
179
for i in range(num_calls):
180
hidden_states = method(
181
hidden_states=hidden_states,
166
- w1=ws,
167
- w2=w2s,
182
+ w1=w1,
183
+ w2=w2,
184
+ w1_scale=w1_scale,
185
+ w2_scale=w2_scale,
186
+ a1_scale=a1_scale,
187
+ a2_scale=a2_scale,
188
gating_output=gating_output[i],
189
topk=2,
190
renormalize=True,
191
inplace=True,
192
override_config=config,
193
+ use_fp8=dtype == "float8",
194
)
195
end_event.record()
196
end_event.synchronize()
200
201
202
if __name__ == "__main__":
182
- sys.exit(main())
203
+ parser = argparse.ArgumentParser(
204
+ prog='benchmark_mixtral_moe',
205
+ description='Benchmark and tune the fused_moe kernel',
206
+ )
207
+ parser.add_argument(
208
+ '--dtype',
209
+ type=str,
210
+ default='auto',
211
+ choices=['float8', 'float16'],
212
+ help='Data type used for fused_moe kernel computations',
213
+ )
214
+ args = parser.parse_args()
215
+ sys.exit(main(args.dtype))