@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a434b07ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Enhance MoE benchmarking & tuning script (#4921)

Woosuk Kwon committed Jun 3, 2024 at 20:06 UTC 3a434b07edc42a7466fb1ac536f3beb9470f9416
3 files changed +346 -253
benchmarks/kernels/benchmark_mixtral_moe.py deleted
-239
@@ -1,239 +0,0 @@
1 -import argparse
2 -import json
3 -import os
4 -import sys
5 -
6 -import torch
7 -import torch.nn.functional as F
8 -import triton
9 -from tqdm import tqdm
10 -
11 -from vllm.model_executor.layers.fused_moe import (fused_moe,
12 - get_config_file_name)
13 -
14 -
15 -def main(model, tp_size, gpu, dtype: str):
16 - os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
17 - method = fused_moe
18 - for bs in [
19 - 1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
20 - 2048, 3072, 4096
21 - ]:
22 - run_grid(bs,
23 - model=model,
24 - method=method,
25 - gpu=gpu,
26 - tp_size=tp_size,
27 - dtype=dtype)
28 -
29 -
30 -def run_grid(bs, model, method, gpu, tp_size, dtype: str):
31 - if model == '8x7B':
32 - d_model = 4096
33 - model_intermediate_size = 14336
34 - num_layers = 32
35 - elif model == '8x22B':
36 - d_model = 6144
37 - model_intermediate_size = 16384
38 - num_layers = 56
39 - else:
40 - raise ValueError(f'Unsupported Mixtral model {model}')
41 - num_total_experts = 8
42 - top_k = 2
43 - # tp_size = 2
44 - num_calls = 100
45 -
46 - num_warmup_trials = 1
47 - num_trials = 1
48 -
49 - configs = []
50 -
51 - for block_size_n in [32, 64, 128, 256]:
52 - for block_size_m in [16, 32, 64, 128, 256]:
53 - for block_size_k in [64, 128, 256]:
54 - for group_size_m in [1, 16, 32, 64]:
55 - for num_warps in [4, 8]:
56 - for num_stages in [2, 3, 4, 5]:
57 - configs.append({
58 - "BLOCK_SIZE_M": block_size_m,
59 - "BLOCK_SIZE_N": block_size_n,
60 - "BLOCK_SIZE_K": block_size_k,
61 - "GROUP_SIZE_M": group_size_m,
62 - "num_warps": num_warps,
63 - "num_stages": num_stages,
64 - })
65 -
66 - best_config = None
67 - best_time_us = 1e20
68 -
69 - print(f'{tp_size=} {bs=}')
70 -
71 - for config in tqdm(configs):
72 - # warmup
73 - try:
74 - for _ in range(num_warmup_trials):
75 - run_timing(
76 - num_calls=num_calls,
77 - bs=bs,
78 - d_model=d_model,
79 - num_total_experts=num_total_experts,
80 - top_k=top_k,
81 - tp_size=tp_size,
82 - model_intermediate_size=model_intermediate_size,
83 - method=method,
84 - config=config,
85 - dtype=dtype,
86 - )
87 - except triton.runtime.autotuner.OutOfResources:
88 - continue
89 -
90 - # trial
91 - for _ in range(num_trials):
92 - kernel_dur_ms = run_timing(
93 - num_calls=num_calls,
94 - bs=bs,
95 - d_model=d_model,
96 - num_total_experts=num_total_experts,
97 - top_k=top_k,
98 - tp_size=tp_size,
99 - model_intermediate_size=model_intermediate_size,
100 - method=method,
101 - config=config,
102 - dtype=dtype,
103 - )
104 -
105 - kernel_dur_us = 1000 * kernel_dur_ms
106 - model_dur_ms = kernel_dur_ms * num_layers
107 -
108 - if kernel_dur_us < best_time_us:
109 - best_config = config
110 - best_time_us = kernel_dur_us
111 -
112 - tqdm.write(
113 - f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
114 - f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
115 - f'{d_model=} {model_intermediate_size=} {num_layers=}')
116 -
117 - print("best_time_us", best_time_us)
118 - print("best_config", best_config)
119 -
120 - # holds Dict[str, Dict[str, int]]
121 - filename = get_config_file_name(num_total_experts,
122 - model_intermediate_size // tp_size,
123 - "float8" if dtype == "float8" else None)
124 - print(f"writing config to file {filename}")
125 - existing_content = {}
126 - if os.path.exists(filename):
127 - with open(filename, "r") as f:
128 - existing_content = json.load(f)
129 - existing_content[str(bs)] = best_config
130 - with open(filename, "w") as f:
131 - json.dump(existing_content, f, indent=4)
132 - f.write("\n")
133 -
134 -
135 -def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
136 - top_k: int, tp_size: int, model_intermediate_size: int, method,
137 - config, dtype: str) -> float:
138 - shard_intermediate_size = model_intermediate_size // tp_size
139 -
140 - hidden_states = torch.rand(
141 - (bs, d_model),
142 - device="cuda:0",
143 - dtype=torch.float16,
144 - )
145 -
146 - w1 = torch.rand(
147 - (num_total_experts, 2 * shard_intermediate_size, d_model),
148 - device=hidden_states.device,
149 - dtype=hidden_states.dtype,
150 - )
151 -
152 - w2 = torch.rand(
153 - (num_total_experts, d_model, shard_intermediate_size),
154 - device=hidden_states.device,
155 - dtype=hidden_states.dtype,
156 - )
157 -
158 - w1_scale = None
159 - w2_scale = None
160 - a1_scale = None
161 - a2_scale = None
162 -
163 - if dtype == "float8":
164 - w1 = w1.to(torch.float8_e4m3fn)
165 - w2 = w2.to(torch.float8_e4m3fn)
166 - w1_scale = torch.ones(num_total_experts,
167 - device=hidden_states.device,
168 - dtype=torch.float32)
169 - w2_scale = torch.ones(num_total_experts,
170 - device=hidden_states.device,
171 - dtype=torch.float32)
172 - a1_scale = torch.ones(1,
173 - device=hidden_states.device,
174 - dtype=torch.float32)
175 - a2_scale = torch.ones(1,
176 - device=hidden_states.device,
177 - dtype=torch.float32)
178 -
179 - gating_output = F.softmax(torch.rand(
180 - (num_calls, bs, num_total_experts),
181 - device=hidden_states.device,
182 - dtype=torch.float32,
183 - ),
184 - dim=-1)
185 -
186 - start_event = torch.cuda.Event(enable_timing=True)
187 - end_event = torch.cuda.Event(enable_timing=True)
188 -
189 - start_event.record()
190 - for i in range(num_calls):
191 - hidden_states = method(
192 - hidden_states=hidden_states,
193 - w1=w1,
194 - w2=w2,
195 - w1_scale=w1_scale,
196 - w2_scale=w2_scale,
197 - a1_scale=a1_scale,
198 - a2_scale=a2_scale,
199 - gating_output=gating_output[i],
200 - topk=2,
201 - renormalize=True,
202 - inplace=True,
203 - override_config=config,
204 - use_fp8=dtype == "float8",
205 - )
206 - end_event.record()
207 - end_event.synchronize()
208 -
209 - dur_ms = start_event.elapsed_time(end_event) / num_calls
210 - return dur_ms
211 -
212 -
213 -if __name__ == "__main__":
214 - parser = argparse.ArgumentParser(
215 - prog='benchmark_mixtral_moe',
216 - description='Benchmark and tune the fused_moe kernel',
217 - )
218 - parser.add_argument(
219 - '--dtype',
220 - type=str,
221 - default='auto',
222 - choices=['float8', 'float16'],
223 - help='Data type used for fused_moe kernel computations',
224 - )
225 - parser.add_argument('--model',
226 - type=str,
227 - default='8x7B',
228 - choices=['8x7B', '8x22B'],
229 - help='The Mixtral model to benchmark')
230 - parser.add_argument('--tp-size',
231 - type=int,
232 - default=2,
233 - help='Tensor paralleli size')
234 - parser.add_argument('--gpu',
235 - type=int,
236 - default=0,
237 - help="GPU ID for benchmarking")
238 - args = parser.parse_args()
239 - sys.exit(main(args.model, args.tp_size, args.gpu, args.dtype))
benchmarks/kernels/benchmark_moe.py new
+319
@@ -0,0 +1,319 @@
1 +import argparse
2 +import time
3 +from datetime import datetime
4 +from typing import Any, Dict, List, Tuple
5 +
6 +import ray
7 +import torch
8 +import triton
9 +from ray.experimental.tqdm_ray import tqdm
10 +from transformers import AutoConfig
11 +
12 +from vllm.model_executor.layers.fused_moe.fused_moe import *
13 +
14 +
15 +def benchmark_config(
16 + config: Dict[str, int],
17 + num_tokens: int,
18 + num_experts: int,
19 + shard_intermediate_size: int,
20 + hidden_size: int,
21 + topk: int,
22 + dtype: torch.dtype,
23 + use_fp8: bool,
24 + num_iters: int = 100,
25 +) -> float:
26 + init_dtype = torch.float16 if use_fp8 else dtype
27 + x = torch.randn(num_tokens, hidden_size, dtype=dtype)
28 + w1 = torch.randn(num_experts,
29 + shard_intermediate_size,
30 + hidden_size,
31 + dtype=init_dtype)
32 + w2 = torch.randn(num_experts,
33 + hidden_size,
34 + shard_intermediate_size // 2,
35 + dtype=init_dtype)
36 + gating_output = torch.randn(num_iters,
37 + num_tokens,
38 + num_experts,
39 + dtype=torch.float32)
40 +
41 + w1_scale = None
42 + w2_scale = None
43 + a1_scale = None
44 + a2_scale = None
45 + if use_fp8:
46 + w1_scale = torch.randn(num_experts, dtype=torch.float32)
47 + w2_scale = torch.randn(num_experts, dtype=torch.float32)
48 + a1_scale = torch.randn(1, dtype=torch.float32)
49 + a2_scale = torch.randn(1, dtype=torch.float32)
50 +
51 + w1 = w1.to(torch.float8_e4m3fn)
52 + w2 = w2.to(torch.float8_e4m3fn)
53 +
54 + input_gating = torch.empty(num_tokens, num_experts, dtype=torch.float32)
55 +
56 + def prepare(i: int):
57 + input_gating.copy_(gating_output[i])
58 +
59 + def run():
60 + fused_moe(
61 + x,
62 + w1,
63 + w2,
64 + input_gating,
65 + topk,
66 + renormalize=True,
67 + inplace=True,
68 + override_config=config,
69 + use_fp8=use_fp8,
70 + w1_scale=w1_scale,
71 + w2_scale=w2_scale,
72 + a1_scale=a1_scale,
73 + a2_scale=a2_scale,
74 + )
75 +
76 + # JIT compilation & warmup
77 + run()
78 + torch.cuda.synchronize()
79 +
80 + # Capture 10 invocations with CUDA graph
81 + graph = torch.cuda.CUDAGraph()
82 + with torch.cuda.graph(graph):
83 + for _ in range(10):
84 + run()
85 + torch.cuda.synchronize()
86 +
87 + # Warmup
88 + for _ in range(5):
89 + graph.replay()
90 + torch.cuda.synchronize()
91 +
92 + start_event = torch.cuda.Event(enable_timing=True)
93 + end_event = torch.cuda.Event(enable_timing=True)
94 +
95 + latencies = []
96 + for i in range(num_iters):
97 + prepare(i)
98 + torch.cuda.synchronize()
99 +
100 + start_event.record()
101 + graph.replay()
102 + end_event.record()
103 + end_event.synchronize()
104 + latencies.append(start_event.elapsed_time(end_event))
105 + avg = sum(latencies) / (num_iters * 10) * 1000 # us
106 + graph.reset()
107 + return avg
108 +
109 +
110 +def get_configs_compute_bound() -> List[Dict[str, int]]:
111 + # Reduced search space for faster tuning.
112 + # TODO(woosuk): Increase the search space and use a performance model to
113 + # prune the search space.
114 + configs = []
115 + for num_stages in [2, 3, 4, 5]:
116 + for block_m in [16, 32, 64, 128, 256]:
117 + for block_k in [64, 128, 256]:
118 + for block_n in [32, 64, 128, 256]:
119 + for num_warps in [4, 8]:
120 + for group_size in [1, 16, 32, 64]:
121 + configs.append({
122 + "BLOCK_SIZE_M": block_m,
123 + "BLOCK_SIZE_N": block_n,
124 + "BLOCK_SIZE_K": block_k,
125 + "GROUP_SIZE_M": group_size,
126 + "num_warps": num_warps,
127 + "num_stages": num_stages,
128 + })
129 + return configs
130 +
131 +
132 +@ray.remote(num_gpus=1)
133 +class BenchmarkWorker:
134 +
135 + def __init__(self, seed: int) -> None:
136 + torch.set_default_device("cuda")
137 + torch.cuda.manual_seed_all(seed)
138 + self.seed = seed
139 +
140 + def benchmark(
141 + self,
142 + num_tokens: int,
143 + num_experts: int,
144 + shard_intermediate_size: int,
145 + hidden_size: int,
146 + topk: int,
147 + dtype: torch.dtype,
148 + use_fp8: bool,
149 + ) -> Tuple[Dict[str, int], float]:
150 + torch.cuda.manual_seed_all(self.seed)
151 +
152 + dtype_str = "float8" if use_fp8 else None
153 + # NOTE(woosuk): The current naming convention uses w2.shape[2], which
154 + # is the intermediate size after silu_and_mul.
155 + op_config = get_moe_configs(num_experts, shard_intermediate_size // 2,
156 + dtype_str)
157 + if op_config is None:
158 + config = get_default_config(num_tokens, num_experts,
159 + shard_intermediate_size, hidden_size,
160 + topk, dtype_str)
161 + else:
162 + config = op_config[min(op_config.keys(),
163 + key=lambda x: abs(x - num_tokens))]
164 + kernel_time = benchmark_config(config, num_tokens, num_experts,
165 + shard_intermediate_size, hidden_size,
166 + topk, dtype, use_fp8)
167 + return config, kernel_time
168 +
169 + def tune(
170 + self,
171 + num_tokens: int,
172 + num_experts: int,
173 + shard_intermediate_size: int,
174 + hidden_size: int,
175 + topk: int,
176 + dtype: torch.dtype,
177 + use_fp8: bool,
178 + search_space: List[Dict[str, int]],
179 + ) -> Dict[str, int]:
180 + best_config = None
181 + best_time = float("inf")
182 + for config in tqdm(search_space):
183 + try:
184 + kernel_time = benchmark_config(config,
185 + num_tokens,
186 + num_experts,
187 + shard_intermediate_size,
188 + hidden_size,
189 + topk,
190 + dtype,
191 + use_fp8,
192 + num_iters=10)
193 + except triton.runtime.autotuner.OutOfResources:
194 + # Some configurations may be invalid and fail to compile.
195 + continue
196 +
197 + if kernel_time < best_time:
198 + best_time = kernel_time
199 + best_config = config
200 + now = datetime.now()
201 + print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}")
202 + return best_config
203 +
204 +
205 +def sort_config(config: Dict[str, int]) -> Dict[str, int]:
206 + return {
207 + "BLOCK_SIZE_M": config["BLOCK_SIZE_M"],
208 + "BLOCK_SIZE_N": config["BLOCK_SIZE_N"],
209 + "BLOCK_SIZE_K": config["BLOCK_SIZE_K"],
210 + "GROUP_SIZE_M": config["GROUP_SIZE_M"],
211 + "num_warps": config["num_warps"],
212 + "num_stages": config["num_stages"],
213 + }
214 +
215 +
216 +def save_configs(
217 + configs: Dict[int, Dict[str, int]],
218 + num_experts: int,
219 + shard_intermediate_size: int,
220 + hidden_size: int,
221 + topk: int,
222 + dtype: torch.dtype,
223 + use_fp8: bool,
224 +) -> None:
225 + dtype_str = "float8" if use_fp8 else None
226 + # NOTE(woosuk): The current naming convention uses w2.shape[2], which
227 + # is the intermediate size after silu_and_mul.
228 + filename = get_config_file_name(num_experts, shard_intermediate_size // 2,
229 + dtype_str)
230 + print(f"Writing best config to {filename}...")
231 + with open(filename, "w") as f:
232 + json.dump(configs, f, indent=4)
233 + f.write("\n")
234 +
235 +
236 +def main(args: argparse.Namespace):
237 + print(args)
238 +
239 + config = AutoConfig.from_pretrained(args.model)
240 + if config.architectures[0] == "DbrxForCausalLM":
241 + E = config.ffn_config.moe_num_experts
242 + topk = config.ffn_config.moe_top_k
243 + intermediate_size = config.ffn_config.ffn_hidden_size
244 + shard_intermediate_size = 2 * intermediate_size // args.tp_size
245 + else:
246 + # Default: Mixtral.
247 + E = config.num_local_experts
248 + topk = config.num_experts_per_tok
249 + intermediate_size = config.intermediate_size
250 + shard_intermediate_size = 2 * intermediate_size // args.tp_size
251 +
252 + hidden_size = config.hidden_size
253 + dtype = config.torch_dtype
254 + use_fp8 = args.dtype == "fp8"
255 +
256 + if args.batch_size is None:
257 + batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]
258 + else:
259 + batch_sizes = [args.batch_size]
260 +
261 + ray.init()
262 + num_gpus = int(ray.available_resources()["GPU"])
263 + workers = [BenchmarkWorker.remote(args.seed) for _ in range(num_gpus)]
264 +
265 + def _distribute(method: str, inputs: List[Any]) -> List[Any]:
266 + outputs = []
267 + worker_idx = 0
268 + for input_args in inputs:
269 + worker = workers[worker_idx]
270 + worker_method = getattr(worker, method)
271 + output = worker_method.remote(*input_args)
272 + outputs.append(output)
273 + worker_idx = (worker_idx + 1) % num_gpus
274 + return ray.get(outputs)
275 +
276 + if args.tune:
277 + search_space = get_configs_compute_bound()
278 + print(f"Start tuning over {len(search_space)} configurations...")
279 +
280 + start = time.time()
281 + configs = _distribute(
282 + "tune", [(batch_size, E, shard_intermediate_size, hidden_size,
283 + topk, dtype, use_fp8, search_space)
284 + for batch_size in batch_sizes])
285 + best_configs = {
286 + M: sort_config(config)
287 + for M, config in zip(batch_sizes, configs)
288 + }
289 + save_configs(best_configs, E, shard_intermediate_size, hidden_size,
290 + topk, dtype, use_fp8)
291 + end = time.time()
292 + print(f"Tuning took {end - start:.2f} seconds")
293 + else:
294 + outputs = _distribute("benchmark",
295 + [(batch_size, E, shard_intermediate_size,
296 + hidden_size, topk, dtype, use_fp8)
297 + for batch_size in batch_sizes])
298 +
299 + for batch_size, (config, kernel_time) in zip(batch_sizes, outputs):
300 + print(f"Batch size: {batch_size}, config: {config}")
301 + print(f"Kernel time: {kernel_time:.2f} us")
302 +
303 +
304 +if __name__ == "__main__":
305 + parser = argparse.ArgumentParser()
306 + parser.add_argument("--model",
307 + type=str,
308 + default="mistralai/Mixtral-8x7B-Instruct-v0.1")
309 + parser.add_argument("--tp-size", "-tp", type=int, default=2)
310 + parser.add_argument("--dtype",
311 + type=str,
312 + choices=["auto", "fp8"],
313 + default="auto")
314 + parser.add_argument("--seed", type=int, default=0)
315 + parser.add_argument("--batch-size", type=int, required=False)
316 + parser.add_argument("--tune", action="store_true")
317 + args = parser.parse_args()
318 +
319 + main(args)
vllm/model_executor/layers/fused_moe/fused_moe.py
+27 -14
@@ -308,6 +308,30 @@ def get_moe_configs(E: int, N: int,
308 return None
309
310
311 +def get_default_config(
312 + M: int,
313 + E: int,
314 + N: int,
315 + K: int,
316 + topk: int,
317 + dtype: Optional[str],
318 +) -> Dict[str, int]:
319 + config = {
320 + 'BLOCK_SIZE_M': 64,
321 + 'BLOCK_SIZE_N': 64,
322 + 'BLOCK_SIZE_K': 32,
323 + 'GROUP_SIZE_M': 8
324 + }
325 + if M <= E:
326 + config = {
327 + 'BLOCK_SIZE_M': 16,
328 + 'BLOCK_SIZE_N': 32,
329 + 'BLOCK_SIZE_K': 64,
330 + 'GROUP_SIZE_M': 1
331 + }
332 + return config
333 +
334 +
335 def fused_topk(
336 hidden_states: torch.Tensor,
337 gating_output: torch.Tensor,
@@ -382,20 +406,9 @@ def fused_experts(hidden_states: torch.Tensor,
406 config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
407 else:
408 # Else use the default config
385 - config = {
386 - 'BLOCK_SIZE_M': 64,
387 - 'BLOCK_SIZE_N': 64,
388 - 'BLOCK_SIZE_K': 32,
389 - 'GROUP_SIZE_M': 8
390 - }
391 -
392 - if M <= E:
393 - config = {
394 - 'BLOCK_SIZE_M': 16,
395 - 'BLOCK_SIZE_N': 32,
396 - 'BLOCK_SIZE_K': 64,
397 - 'GROUP_SIZE_M': 1
398 - }
409 + config = get_default_config(M, E, N, w1.shape[2],
410 + topk_ids.shape[1],
411 + "float8" if use_fp8 else None)
412
413 intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
414 device=hidden_states.device,