A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Enhance MoE benchmarking & tuning script (#4921)
Woosuk Kwon committed
Jun 3, 2024 at 20:06 UTC
3a434b07edc42a7466fb1ac536f3beb9470f9416
3 files changed
+346
-253
benchmarks/kernels/benchmark_mixtral_moe.py
deleted
-239
@@ -1,239 +0,0 @@
1
-import argparse
2
-import json
3
-import os
4
-import sys
5
-
6
-import torch
7
-import torch.nn.functional as F
8
-import triton
9
-from tqdm import tqdm
10
-
11
-from vllm.model_executor.layers.fused_moe import (fused_moe,
12
- get_config_file_name)
13
-
14
-
15
-def main(model, tp_size, gpu, dtype: str):
16
- os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
17
- method = fused_moe
18
- for bs in [
19
- 1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
20
- 2048, 3072, 4096
21
- ]:
22
- run_grid(bs,
23
- model=model,
24
- method=method,
25
- gpu=gpu,
26
- tp_size=tp_size,
27
- dtype=dtype)
28
-
29
-
30
-def run_grid(bs, model, method, gpu, tp_size, dtype: str):
31
- if model == '8x7B':
32
- d_model = 4096
33
- model_intermediate_size = 14336
34
- num_layers = 32
35
- elif model == '8x22B':
36
- d_model = 6144
37
- model_intermediate_size = 16384
38
- num_layers = 56
39
- else:
40
- raise ValueError(f'Unsupported Mixtral model {model}')
41
- num_total_experts = 8
42
- top_k = 2
43
- # tp_size = 2
44
- num_calls = 100
45
-
46
- num_warmup_trials = 1
47
- num_trials = 1
48
-
49
- configs = []
50
-
51
- for block_size_n in [32, 64, 128, 256]:
52
- for block_size_m in [16, 32, 64, 128, 256]:
53
- for block_size_k in [64, 128, 256]:
54
- for group_size_m in [1, 16, 32, 64]:
55
- for num_warps in [4, 8]:
56
- for num_stages in [2, 3, 4, 5]:
57
- configs.append({
58
- "BLOCK_SIZE_M": block_size_m,
59
- "BLOCK_SIZE_N": block_size_n,
60
- "BLOCK_SIZE_K": block_size_k,
61
- "GROUP_SIZE_M": group_size_m,
62
- "num_warps": num_warps,
63
- "num_stages": num_stages,
64
- })
65
-
66
- best_config = None
67
- best_time_us = 1e20
68
-
69
- print(f'{tp_size=} {bs=}')
70
-
71
- for config in tqdm(configs):
72
- # warmup
73
- try:
74
- for _ in range(num_warmup_trials):
75
- run_timing(
76
- num_calls=num_calls,
77
- bs=bs,
78
- d_model=d_model,
79
- num_total_experts=num_total_experts,
80
- top_k=top_k,
81
- tp_size=tp_size,
82
- model_intermediate_size=model_intermediate_size,
83
- method=method,
84
- config=config,
85
- dtype=dtype,
86
- )
87
- except triton.runtime.autotuner.OutOfResources:
88
- continue
89
-
90
- # trial
91
- for _ in range(num_trials):
92
- kernel_dur_ms = run_timing(
93
- num_calls=num_calls,
94
- bs=bs,
95
- d_model=d_model,
96
- num_total_experts=num_total_experts,
97
- top_k=top_k,
98
- tp_size=tp_size,
99
- model_intermediate_size=model_intermediate_size,
100
- method=method,
101
- config=config,
102
- dtype=dtype,
103
- )
104
-
105
- kernel_dur_us = 1000 * kernel_dur_ms
106
- model_dur_ms = kernel_dur_ms * num_layers
107
-
108
- if kernel_dur_us < best_time_us:
109
- best_config = config
110
- best_time_us = kernel_dur_us
111
-
112
- tqdm.write(
113
- f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
114
- f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
115
- f'{d_model=} {model_intermediate_size=} {num_layers=}')
116
-
117
- print("best_time_us", best_time_us)
118
- print("best_config", best_config)
119
-
120
- # holds Dict[str, Dict[str, int]]
121
- filename = get_config_file_name(num_total_experts,
122
- model_intermediate_size // tp_size,
123
- "float8" if dtype == "float8" else None)
124
- print(f"writing config to file {filename}")
125
- existing_content = {}
126
- if os.path.exists(filename):
127
- with open(filename, "r") as f:
128
- existing_content = json.load(f)
129
- existing_content[str(bs)] = best_config
130
- with open(filename, "w") as f:
131
- json.dump(existing_content, f, indent=4)
132
- f.write("\n")
133
-
134
-
135
-def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
136
- top_k: int, tp_size: int, model_intermediate_size: int, method,
137
- config, dtype: str) -> float:
138
- shard_intermediate_size = model_intermediate_size // tp_size
139
-
140
- hidden_states = torch.rand(
141
- (bs, d_model),
142
- device="cuda:0",
143
- dtype=torch.float16,
144
- )
145
-
146
- w1 = torch.rand(
147
- (num_total_experts, 2 * shard_intermediate_size, d_model),
148
- device=hidden_states.device,
149
- dtype=hidden_states.dtype,
150
- )
151
-
152
- w2 = torch.rand(
153
- (num_total_experts, d_model, shard_intermediate_size),
154
- device=hidden_states.device,
155
- dtype=hidden_states.dtype,
156
- )
157
-
158
- w1_scale = None
159
- w2_scale = None
160
- a1_scale = None
161
- a2_scale = None
162
-
163
- if dtype == "float8":
164
- w1 = w1.to(torch.float8_e4m3fn)
165
- w2 = w2.to(torch.float8_e4m3fn)
166
- w1_scale = torch.ones(num_total_experts,
167
- device=hidden_states.device,
168
- dtype=torch.float32)
169
- w2_scale = torch.ones(num_total_experts,
170
- device=hidden_states.device,
171
- dtype=torch.float32)
172
- a1_scale = torch.ones(1,
173
- device=hidden_states.device,
174
- dtype=torch.float32)
175
- a2_scale = torch.ones(1,
176
- device=hidden_states.device,
177
- dtype=torch.float32)
178
-
179
- gating_output = F.softmax(torch.rand(
180
- (num_calls, bs, num_total_experts),
181
- device=hidden_states.device,
182
- dtype=torch.float32,
183
- ),
184
- dim=-1)
185
-
186
- start_event = torch.cuda.Event(enable_timing=True)
187
- end_event = torch.cuda.Event(enable_timing=True)
188
-
189
- start_event.record()
190
- for i in range(num_calls):
191
- hidden_states = method(
192
- hidden_states=hidden_states,
193
- w1=w1,
194
- w2=w2,
195
- w1_scale=w1_scale,
196
- w2_scale=w2_scale,
197
- a1_scale=a1_scale,
198
- a2_scale=a2_scale,
199
- gating_output=gating_output[i],
200
- topk=2,
201
- renormalize=True,
202
- inplace=True,
203
- override_config=config,
204
- use_fp8=dtype == "float8",
205
- )
206
- end_event.record()
207
- end_event.synchronize()
208
-
209
- dur_ms = start_event.elapsed_time(end_event) / num_calls
210
- return dur_ms
211
-
212
-
213
-if __name__ == "__main__":
214
- parser = argparse.ArgumentParser(
215
- prog='benchmark_mixtral_moe',
216
- description='Benchmark and tune the fused_moe kernel',
217
- )
218
- parser.add_argument(
219
- '--dtype',
220
- type=str,
221
- default='auto',
222
- choices=['float8', 'float16'],
223
- help='Data type used for fused_moe kernel computations',
224
- )
225
- parser.add_argument('--model',
226
- type=str,
227
- default='8x7B',
228
- choices=['8x7B', '8x22B'],
229
- help='The Mixtral model to benchmark')
230
- parser.add_argument('--tp-size',
231
- type=int,
232
- default=2,
233
- help='Tensor paralleli size')
234
- parser.add_argument('--gpu',
235
- type=int,
236
- default=0,
237
- help="GPU ID for benchmarking")
238
- args = parser.parse_args()
239
- sys.exit(main(args.model, args.tp_size, args.gpu, args.dtype))
benchmarks/kernels/benchmark_moe.py
new
+319
@@ -0,0 +1,319 @@
1
+import argparse
2
+import time
3
+from datetime import datetime
4
+from typing import Any, Dict, List, Tuple
5
+
6
+import ray
7
+import torch
8
+import triton
9
+from ray.experimental.tqdm_ray import tqdm
10
+from transformers import AutoConfig
11
+
12
+from vllm.model_executor.layers.fused_moe.fused_moe import *
13
+
14
+
15
+def benchmark_config(
16
+ config: Dict[str, int],
17
+ num_tokens: int,
18
+ num_experts: int,
19
+ shard_intermediate_size: int,
20
+ hidden_size: int,
21
+ topk: int,
22
+ dtype: torch.dtype,
23
+ use_fp8: bool,
24
+ num_iters: int = 100,
25
+) -> float:
26
+ init_dtype = torch.float16 if use_fp8 else dtype
27
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype)
28
+ w1 = torch.randn(num_experts,
29
+ shard_intermediate_size,
30
+ hidden_size,
31
+ dtype=init_dtype)
32
+ w2 = torch.randn(num_experts,
33
+ hidden_size,
34
+ shard_intermediate_size // 2,
35
+ dtype=init_dtype)
36
+ gating_output = torch.randn(num_iters,
37
+ num_tokens,
38
+ num_experts,
39
+ dtype=torch.float32)
40
+
41
+ w1_scale = None
42
+ w2_scale = None
43
+ a1_scale = None
44
+ a2_scale = None
45
+ if use_fp8:
46
+ w1_scale = torch.randn(num_experts, dtype=torch.float32)
47
+ w2_scale = torch.randn(num_experts, dtype=torch.float32)
48
+ a1_scale = torch.randn(1, dtype=torch.float32)
49
+ a2_scale = torch.randn(1, dtype=torch.float32)
50
+
51
+ w1 = w1.to(torch.float8_e4m3fn)
52
+ w2 = w2.to(torch.float8_e4m3fn)
53
+
54
+ input_gating = torch.empty(num_tokens, num_experts, dtype=torch.float32)
55
+
56
+ def prepare(i: int):
57
+ input_gating.copy_(gating_output[i])
58
+
59
+ def run():
60
+ fused_moe(
61
+ x,
62
+ w1,
63
+ w2,
64
+ input_gating,
65
+ topk,
66
+ renormalize=True,
67
+ inplace=True,
68
+ override_config=config,
69
+ use_fp8=use_fp8,
70
+ w1_scale=w1_scale,
71
+ w2_scale=w2_scale,
72
+ a1_scale=a1_scale,
73
+ a2_scale=a2_scale,
74
+ )
75
+
76
+ # JIT compilation & warmup
77
+ run()
78
+ torch.cuda.synchronize()
79
+
80
+ # Capture 10 invocations with CUDA graph
81
+ graph = torch.cuda.CUDAGraph()
82
+ with torch.cuda.graph(graph):
83
+ for _ in range(10):
84
+ run()
85
+ torch.cuda.synchronize()
86
+
87
+ # Warmup
88
+ for _ in range(5):
89
+ graph.replay()
90
+ torch.cuda.synchronize()
91
+
92
+ start_event = torch.cuda.Event(enable_timing=True)
93
+ end_event = torch.cuda.Event(enable_timing=True)
94
+
95
+ latencies = []
96
+ for i in range(num_iters):
97
+ prepare(i)
98
+ torch.cuda.synchronize()
99
+
100
+ start_event.record()
101
+ graph.replay()
102
+ end_event.record()
103
+ end_event.synchronize()
104
+ latencies.append(start_event.elapsed_time(end_event))
105
+ avg = sum(latencies) / (num_iters * 10) * 1000 # us
106
+ graph.reset()
107
+ return avg
108
+
109
+
110
+def get_configs_compute_bound() -> List[Dict[str, int]]:
111
+ # Reduced search space for faster tuning.
112
+ # TODO(woosuk): Increase the search space and use a performance model to
113
+ # prune the search space.
114
+ configs = []
115
+ for num_stages in [2, 3, 4, 5]:
116
+ for block_m in [16, 32, 64, 128, 256]:
117
+ for block_k in [64, 128, 256]:
118
+ for block_n in [32, 64, 128, 256]:
119
+ for num_warps in [4, 8]:
120
+ for group_size in [1, 16, 32, 64]:
121
+ configs.append({
122
+ "BLOCK_SIZE_M": block_m,
123
+ "BLOCK_SIZE_N": block_n,
124
+ "BLOCK_SIZE_K": block_k,
125
+ "GROUP_SIZE_M": group_size,
126
+ "num_warps": num_warps,
127
+ "num_stages": num_stages,
128
+ })
129
+ return configs
130
+
131
+
132
+@ray.remote(num_gpus=1)
133
+class BenchmarkWorker:
134
+
135
+ def __init__(self, seed: int) -> None:
136
+ torch.set_default_device("cuda")
137
+ torch.cuda.manual_seed_all(seed)
138
+ self.seed = seed
139
+
140
+ def benchmark(
141
+ self,
142
+ num_tokens: int,
143
+ num_experts: int,
144
+ shard_intermediate_size: int,
145
+ hidden_size: int,
146
+ topk: int,
147
+ dtype: torch.dtype,
148
+ use_fp8: bool,
149
+ ) -> Tuple[Dict[str, int], float]:
150
+ torch.cuda.manual_seed_all(self.seed)
151
+
152
+ dtype_str = "float8" if use_fp8 else None
153
+ # NOTE(woosuk): The current naming convention uses w2.shape[2], which
154
+ # is the intermediate size after silu_and_mul.
155
+ op_config = get_moe_configs(num_experts, shard_intermediate_size // 2,
156
+ dtype_str)
157
+ if op_config is None:
158
+ config = get_default_config(num_tokens, num_experts,
159
+ shard_intermediate_size, hidden_size,
160
+ topk, dtype_str)
161
+ else:
162
+ config = op_config[min(op_config.keys(),
163
+ key=lambda x: abs(x - num_tokens))]
164
+ kernel_time = benchmark_config(config, num_tokens, num_experts,
165
+ shard_intermediate_size, hidden_size,
166
+ topk, dtype, use_fp8)
167
+ return config, kernel_time
168
+
169
+ def tune(
170
+ self,
171
+ num_tokens: int,
172
+ num_experts: int,
173
+ shard_intermediate_size: int,
174
+ hidden_size: int,
175
+ topk: int,
176
+ dtype: torch.dtype,
177
+ use_fp8: bool,
178
+ search_space: List[Dict[str, int]],
179
+ ) -> Dict[str, int]:
180
+ best_config = None
181
+ best_time = float("inf")
182
+ for config in tqdm(search_space):
183
+ try:
184
+ kernel_time = benchmark_config(config,
185
+ num_tokens,
186
+ num_experts,
187
+ shard_intermediate_size,
188
+ hidden_size,
189
+ topk,
190
+ dtype,
191
+ use_fp8,
192
+ num_iters=10)
193
+ except triton.runtime.autotuner.OutOfResources:
194
+ # Some configurations may be invalid and fail to compile.
195
+ continue
196
+
197
+ if kernel_time < best_time:
198
+ best_time = kernel_time
199
+ best_config = config
200
+ now = datetime.now()
201
+ print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}")
202
+ return best_config
203
+
204
+
205
+def sort_config(config: Dict[str, int]) -> Dict[str, int]:
206
+ return {
207
+ "BLOCK_SIZE_M": config["BLOCK_SIZE_M"],
208
+ "BLOCK_SIZE_N": config["BLOCK_SIZE_N"],
209
+ "BLOCK_SIZE_K": config["BLOCK_SIZE_K"],
210
+ "GROUP_SIZE_M": config["GROUP_SIZE_M"],
211
+ "num_warps": config["num_warps"],
212
+ "num_stages": config["num_stages"],
213
+ }
214
+
215
+
216
+def save_configs(
217
+ configs: Dict[int, Dict[str, int]],
218
+ num_experts: int,
219
+ shard_intermediate_size: int,
220
+ hidden_size: int,
221
+ topk: int,
222
+ dtype: torch.dtype,
223
+ use_fp8: bool,
224
+) -> None:
225
+ dtype_str = "float8" if use_fp8 else None
226
+ # NOTE(woosuk): The current naming convention uses w2.shape[2], which
227
+ # is the intermediate size after silu_and_mul.
228
+ filename = get_config_file_name(num_experts, shard_intermediate_size // 2,
229
+ dtype_str)
230
+ print(f"Writing best config to {filename}...")
231
+ with open(filename, "w") as f:
232
+ json.dump(configs, f, indent=4)
233
+ f.write("\n")
234
+
235
+
236
+def main(args: argparse.Namespace):
237
+ print(args)
238
+
239
+ config = AutoConfig.from_pretrained(args.model)
240
+ if config.architectures[0] == "DbrxForCausalLM":
241
+ E = config.ffn_config.moe_num_experts
242
+ topk = config.ffn_config.moe_top_k
243
+ intermediate_size = config.ffn_config.ffn_hidden_size
244
+ shard_intermediate_size = 2 * intermediate_size // args.tp_size
245
+ else:
246
+ # Default: Mixtral.
247
+ E = config.num_local_experts
248
+ topk = config.num_experts_per_tok
249
+ intermediate_size = config.intermediate_size
250
+ shard_intermediate_size = 2 * intermediate_size // args.tp_size
251
+
252
+ hidden_size = config.hidden_size
253
+ dtype = config.torch_dtype
254
+ use_fp8 = args.dtype == "fp8"
255
+
256
+ if args.batch_size is None:
257
+ batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]
258
+ else:
259
+ batch_sizes = [args.batch_size]
260
+
261
+ ray.init()
262
+ num_gpus = int(ray.available_resources()["GPU"])
263
+ workers = [BenchmarkWorker.remote(args.seed) for _ in range(num_gpus)]
264
+
265
+ def _distribute(method: str, inputs: List[Any]) -> List[Any]:
266
+ outputs = []
267
+ worker_idx = 0
268
+ for input_args in inputs:
269
+ worker = workers[worker_idx]
270
+ worker_method = getattr(worker, method)
271
+ output = worker_method.remote(*input_args)
272
+ outputs.append(output)
273
+ worker_idx = (worker_idx + 1) % num_gpus
274
+ return ray.get(outputs)
275
+
276
+ if args.tune:
277
+ search_space = get_configs_compute_bound()
278
+ print(f"Start tuning over {len(search_space)} configurations...")
279
+
280
+ start = time.time()
281
+ configs = _distribute(
282
+ "tune", [(batch_size, E, shard_intermediate_size, hidden_size,
283
+ topk, dtype, use_fp8, search_space)
284
+ for batch_size in batch_sizes])
285
+ best_configs = {
286
+ M: sort_config(config)
287
+ for M, config in zip(batch_sizes, configs)
288
+ }
289
+ save_configs(best_configs, E, shard_intermediate_size, hidden_size,
290
+ topk, dtype, use_fp8)
291
+ end = time.time()
292
+ print(f"Tuning took {end - start:.2f} seconds")
293
+ else:
294
+ outputs = _distribute("benchmark",
295
+ [(batch_size, E, shard_intermediate_size,
296
+ hidden_size, topk, dtype, use_fp8)
297
+ for batch_size in batch_sizes])
298
+
299
+ for batch_size, (config, kernel_time) in zip(batch_sizes, outputs):
300
+ print(f"Batch size: {batch_size}, config: {config}")
301
+ print(f"Kernel time: {kernel_time:.2f} us")
302
+
303
+
304
+if __name__ == "__main__":
305
+ parser = argparse.ArgumentParser()
306
+ parser.add_argument("--model",
307
+ type=str,
308
+ default="mistralai/Mixtral-8x7B-Instruct-v0.1")
309
+ parser.add_argument("--tp-size", "-tp", type=int, default=2)
310
+ parser.add_argument("--dtype",
311
+ type=str,
312
+ choices=["auto", "fp8"],
313
+ default="auto")
314
+ parser.add_argument("--seed", type=int, default=0)
315
+ parser.add_argument("--batch-size", type=int, required=False)
316
+ parser.add_argument("--tune", action="store_true")
317
+ args = parser.parse_args()
318
+
319
+ main(args)
vllm/model_executor/layers/fused_moe/fused_moe.py
+27
-14
@@ -308,6 +308,30 @@ def get_moe_configs(E: int, N: int,
308
return None
309
310
311
+def get_default_config(
312
+ M: int,
313
+ E: int,
314
+ N: int,
315
+ K: int,
316
+ topk: int,
317
+ dtype: Optional[str],
318
+) -> Dict[str, int]:
319
+ config = {
320
+ 'BLOCK_SIZE_M': 64,
321
+ 'BLOCK_SIZE_N': 64,
322
+ 'BLOCK_SIZE_K': 32,
323
+ 'GROUP_SIZE_M': 8
324
+ }
325
+ if M <= E:
326
+ config = {
327
+ 'BLOCK_SIZE_M': 16,
328
+ 'BLOCK_SIZE_N': 32,
329
+ 'BLOCK_SIZE_K': 64,
330
+ 'GROUP_SIZE_M': 1
331
+ }
332
+ return config
333
+
334
+
335
def fused_topk(
336
hidden_states: torch.Tensor,
337
gating_output: torch.Tensor,
@@ -382,20 +406,9 @@ def fused_experts(hidden_states: torch.Tensor,
406
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
407
else:
408
# Else use the default config
385
- config = {
386
- 'BLOCK_SIZE_M': 64,
387
- 'BLOCK_SIZE_N': 64,
388
- 'BLOCK_SIZE_K': 32,
389
- 'GROUP_SIZE_M': 8
390
- }
391
-
392
- if M <= E:
393
- config = {
394
- 'BLOCK_SIZE_M': 16,
395
- 'BLOCK_SIZE_N': 32,
396
- 'BLOCK_SIZE_K': 64,
397
- 'GROUP_SIZE_M': 1
398
- }
409
+ config = get_default_config(M, E, N, w1.shape[2],
410
+ topk_ids.shape[1],
411
+ "float8" if use_fp8 else None)
412
413
intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
414
device=hidden_states.device,