@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24bb4fe432
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Update fused_moe tuning script for FP8 (#4457)

This PR updates the tuning script for the fused_moe kernel to support FP8 and also adds configurations for TP4. Note that for the configuration I removed num_warps and num_stages for small batch sizes since that improved performance and brought the benchmarks on par with the numbers before in that regime to make sure this is a strict improvement over the status quo. All the numbers below are for mistralai/Mixtral-8x7B-Instruct-v0.1, 1000 input and 50 output tokens. Before this PR (with static activation scaling): qps = 1: 9.8 ms ITL, 0.49s e2e latency qps = 2: 9.7 ms ITL, 0.49s e2e latency qps = 4: 10.1 ms ITL, 0.52s e2e latency qps = 6: 11.9 ms ITL, 0.59s e2e latency qps = 8: 14.0 ms ITL, 0.70s e2e latency qps = 10: 15.7 ms ITL, 0.79s e2e latency After this PR (with static activation scaling): qps = 1: 9.8 ms ITL, 0.49s e2e latency qps = 2: 9.7 ms ITL, 0.49s e2e latency qps = 4: 10.2 ms ITL, 0.53s e2e latency qps = 6: 11.9 ms ITL, 0.59s e2e latency qps = 8: 11.9 ms ITL, 0.59s e2e latency qps = 10: 12.1 ms ITL, 0.61s e2e latency

Philipp Moritz committed May 1, 2024 at 11:47 UTC 24bb4fe432fffeccf7a27270ee70aff1b1b8a89a
2 files changed +211 -38
benchmarks/kernels/benchmark_mixtral_moe.py
+71 -38
@@ -1,3 +1,4 @@
1 +import argparse
2 import json
3 import os
4 import sys
@@ -5,6 +6,7 @@ import sys
6 import torch
7 import torch.nn.functional as F
8 import triton
9 +from tqdm import tqdm
10
11 from vllm.model_executor.layers.fused_moe import (fused_moe,
12 get_config_file_name)
@@ -12,16 +14,16 @@ from vllm.model_executor.layers.fused_moe import (fused_moe,
14 os.environ['CUDA_VISIBLE_DEVICES'] = '0'
15
16
15 -def main():
17 +def main(dtype: str):
18 method = fused_moe
19 for bs in [
20 1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
21 2048, 3072, 4096
22 ]:
21 - run_grid(bs, method=method)
23 + run_grid(bs, method=method, dtype=dtype)
24
25
24 -def run_grid(bs, method):
26 +def run_grid(bs, method, dtype: str):
27 d_model = 4096
28 num_total_experts = 8
29 top_k = 2
@@ -34,39 +36,29 @@ def run_grid(bs, method):
36 num_trials = 1
37
38 configs = []
37 - if bs <= 16:
38 - BLOCK_SIZES_M = [16]
39 - elif bs <= 32:
40 - BLOCK_SIZES_M = [16, 32]
41 - elif bs <= 64:
42 - BLOCK_SIZES_M = [16, 32, 64]
43 - elif bs <= 128:
44 - BLOCK_SIZES_M = [16, 32, 64, 128]
45 - else:
46 - BLOCK_SIZES_M = [16, 32, 64, 128, 256]
39
40 for block_size_n in [32, 64, 128, 256]:
49 - for block_size_m in BLOCK_SIZES_M:
41 + for block_size_m in [16, 32, 64, 128, 256]:
42 for block_size_k in [64, 128, 256]:
43 for group_size_m in [1, 16, 32, 64]:
44 for num_warps in [4, 8]:
53 - configs.append({
54 - "BLOCK_SIZE_M": block_size_m,
55 - "BLOCK_SIZE_N": block_size_n,
56 - "BLOCK_SIZE_K": block_size_k,
57 - "GROUP_SIZE_M": group_size_m,
58 - "num_warps": num_warps,
59 - "num_stages": 4,
60 - })
45 + for num_stages in [2, 3, 4, 5]:
46 + configs.append({
47 + "BLOCK_SIZE_M": block_size_m,
48 + "BLOCK_SIZE_N": block_size_n,
49 + "BLOCK_SIZE_K": block_size_k,
50 + "GROUP_SIZE_M": group_size_m,
51 + "num_warps": num_warps,
52 + "num_stages": num_stages,
53 + })
54
55 best_config = None
56 best_time_us = 1e20
57
65 - for config in configs:
66 - print(f'{tp_size=} {bs=}')
67 - print(f'{config}')
58 + print(f'{tp_size=} {bs=}')
59 +
60 + for config in tqdm(configs):
61 # warmup
69 - print('warming up')
62 try:
63 for _ in range(num_warmup_trials):
64 run_timing(
@@ -79,12 +71,12 @@ def run_grid(bs, method):
71 model_intermediate_size=model_intermediate_size,
72 method=method,
73 config=config,
74 + dtype=dtype,
75 )
76 except triton.runtime.autotuner.OutOfResources:
77 continue
78
79 # trial
87 - print('benchmarking')
80 for _ in range(num_trials):
81 kernel_dur_ms = run_timing(
82 num_calls=num_calls,
@@ -96,6 +88,7 @@ def run_grid(bs, method):
88 model_intermediate_size=model_intermediate_size,
89 method=method,
90 config=config,
91 + dtype=dtype,
92 )
93
94 kernel_dur_us = 1000 * kernel_dur_ms
@@ -105,16 +98,18 @@ def run_grid(bs, method):
98 best_config = config
99 best_time_us = kernel_dur_us
100
108 - print(f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
109 - f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
110 - f'{d_model=} {model_intermediate_size=} {num_layers=}')
101 + tqdm.write(
102 + f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
103 + f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
104 + f'{d_model=} {model_intermediate_size=} {num_layers=}')
105
106 print("best_time_us", best_time_us)
107 print("best_config", best_config)
108
109 # holds Dict[str, Dict[str, int]]
110 filename = get_config_file_name(num_total_experts,
117 - model_intermediate_size // tp_size)
111 + model_intermediate_size // tp_size,
112 + "float8" if dtype == "float8" else None)
113 print(f"writing config to file {filename}")
114 existing_content = {}
115 if os.path.exists(filename):
@@ -128,27 +123,48 @@ def run_grid(bs, method):
123
124 def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
125 top_k: int, tp_size: int, model_intermediate_size: int, method,
131 - config) -> float:
126 + config, dtype: str) -> float:
127 shard_intermediate_size = model_intermediate_size // tp_size
128
129 hidden_states = torch.rand(
130 (bs, d_model),
131 device="cuda:0",
137 - dtype=torch.bfloat16,
132 + dtype=torch.float16,
133 )
134
140 - ws = torch.rand(
135 + w1 = torch.rand(
136 (num_total_experts, 2 * shard_intermediate_size, d_model),
137 device=hidden_states.device,
138 dtype=hidden_states.dtype,
139 )
140
146 - w2s = torch.rand(
141 + w2 = torch.rand(
142 (num_total_experts, d_model, shard_intermediate_size),
143 device=hidden_states.device,
144 dtype=hidden_states.dtype,
145 )
146
147 + w1_scale = None
148 + w2_scale = None
149 + a1_scale = None
150 + a2_scale = None
151 +
152 + if dtype == "float8":
153 + w1 = w1.to(torch.float8_e4m3fn)
154 + w2 = w2.to(torch.float8_e4m3fn)
155 + w1_scale = torch.ones(num_total_experts,
156 + device=hidden_states.device,
157 + dtype=torch.float32)
158 + w2_scale = torch.ones(num_total_experts,
159 + device=hidden_states.device,
160 + dtype=torch.float32)
161 + a1_scale = torch.ones(1,
162 + device=hidden_states.device,
163 + dtype=torch.float32)
164 + a2_scale = torch.ones(1,
165 + device=hidden_states.device,
166 + dtype=torch.float32)
167 +
168 gating_output = F.softmax(torch.rand(
169 (num_calls, bs, num_total_experts),
170 device=hidden_states.device,
@@ -163,13 +179,18 @@ def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
179 for i in range(num_calls):
180 hidden_states = method(
181 hidden_states=hidden_states,
166 - w1=ws,
167 - w2=w2s,
182 + w1=w1,
183 + w2=w2,
184 + w1_scale=w1_scale,
185 + w2_scale=w2_scale,
186 + a1_scale=a1_scale,
187 + a2_scale=a2_scale,
188 gating_output=gating_output[i],
189 topk=2,
190 renormalize=True,
191 inplace=True,
192 override_config=config,
193 + use_fp8=dtype == "float8",
194 )
195 end_event.record()
196 end_event.synchronize()
@@ -179,4 +200,16 @@ def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
200
201
202 if __name__ == "__main__":
182 - sys.exit(main())
203 + parser = argparse.ArgumentParser(
204 + prog='benchmark_mixtral_moe',
205 + description='Benchmark and tune the fused_moe kernel',
206 + )
207 + parser.add_argument(
208 + '--dtype',
209 + type=str,
210 + default='auto',
211 + choices=['float8', 'float16'],
212 + help='Data type used for fused_moe kernel computations',
213 + )
214 + args = parser.parse_args()
215 + sys.exit(main(args.dtype))
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json new
+140
@@ -0,0 +1,140 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1
7 + },
8 + "2": {
9 + "BLOCK_SIZE_M": 16,
10 + "BLOCK_SIZE_N": 32,
11 + "BLOCK_SIZE_K": 64,
12 + "GROUP_SIZE_M": 1
13 + },
14 + "4": {
15 + "BLOCK_SIZE_M": 16,
16 + "BLOCK_SIZE_N": 32,
17 + "BLOCK_SIZE_K": 64,
18 + "GROUP_SIZE_M": 1
19 + },
20 + "8": {
21 + "BLOCK_SIZE_M": 64,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 256,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 8,
26 + "num_stages": 5
27 + },
28 + "16": {
29 + "BLOCK_SIZE_M": 64,
30 + "BLOCK_SIZE_N": 128,
31 + "BLOCK_SIZE_K": 64,
32 + "GROUP_SIZE_M": 16,
33 + "num_warps": 4,
34 + "num_stages": 5
35 + },
36 + "24": {
37 + "BLOCK_SIZE_M": 128,
38 + "BLOCK_SIZE_N": 64,
39 + "BLOCK_SIZE_K": 128,
40 + "GROUP_SIZE_M": 1,
41 + "num_warps": 8,
42 + "num_stages": 5
43 + },
44 + "32": {
45 + "BLOCK_SIZE_M": 64,
46 + "BLOCK_SIZE_N": 256,
47 + "BLOCK_SIZE_K": 128,
48 + "GROUP_SIZE_M": 16,
49 + "num_warps": 8,
50 + "num_stages": 4
51 + },
52 + "48": {
53 + "BLOCK_SIZE_M": 64,
54 + "BLOCK_SIZE_N": 256,
55 + "BLOCK_SIZE_K": 128,
56 + "GROUP_SIZE_M": 64,
57 + "num_warps": 8,
58 + "num_stages": 3
59 + },
60 + "64": {
61 + "BLOCK_SIZE_M": 64,
62 + "BLOCK_SIZE_N": 256,
63 + "BLOCK_SIZE_K": 64,
64 + "GROUP_SIZE_M": 64,
65 + "num_warps": 4,
66 + "num_stages": 4
67 + },
68 + "96": {
69 + "BLOCK_SIZE_M": 128,
70 + "BLOCK_SIZE_N": 64,
71 + "BLOCK_SIZE_K": 256,
72 + "GROUP_SIZE_M": 32,
73 + "num_warps": 8,
74 + "num_stages": 2
75 + },
76 + "128": {
77 + "BLOCK_SIZE_M": 128,
78 + "BLOCK_SIZE_N": 64,
79 + "BLOCK_SIZE_K": 128,
80 + "GROUP_SIZE_M": 32,
81 + "num_warps": 4,
82 + "num_stages": 3
83 + },
84 + "256": {
85 + "BLOCK_SIZE_M": 128,
86 + "BLOCK_SIZE_N": 32,
87 + "BLOCK_SIZE_K": 128,
88 + "GROUP_SIZE_M": 16,
89 + "num_warps": 4,
90 + "num_stages": 5
91 + },
92 + "512": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 128,
95 + "BLOCK_SIZE_K": 256,
96 + "GROUP_SIZE_M": 64,
97 + "num_warps": 4,
98 + "num_stages": 2
99 + },
100 + "1024": {
101 + "BLOCK_SIZE_M": 128,
102 + "BLOCK_SIZE_N": 256,
103 + "BLOCK_SIZE_K": 128,
104 + "GROUP_SIZE_M": 64,
105 + "num_warps": 8,
106 + "num_stages": 4
107 + },
108 + "1536": {
109 + "BLOCK_SIZE_M": 128,
110 + "BLOCK_SIZE_N": 256,
111 + "BLOCK_SIZE_K": 128,
112 + "GROUP_SIZE_M": 32,
113 + "num_warps": 8,
114 + "num_stages": 4
115 + },
116 + "2048": {
117 + "BLOCK_SIZE_M": 128,
118 + "BLOCK_SIZE_N": 256,
119 + "BLOCK_SIZE_K": 128,
120 + "GROUP_SIZE_M": 64,
121 + "num_warps": 8,
122 + "num_stages": 4
123 + },
124 + "3072": {
125 + "BLOCK_SIZE_M": 128,
126 + "BLOCK_SIZE_N": 256,
127 + "BLOCK_SIZE_K": 128,
128 + "GROUP_SIZE_M": 32,
129 + "num_warps": 8,
130 + "num_stages": 4
131 + },
132 + "4096": {
133 + "BLOCK_SIZE_M": 128,
134 + "BLOCK_SIZE_N": 256,
135 + "BLOCK_SIZE_K": 128,
136 + "GROUP_SIZE_M": 16,
137 + "num_warps": 8,
138 + "num_stages": 4
139 + }
140 +}