@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cfc15a1031
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Optimize Triton MoE Kernel (#2979)

Co-authored-by: Cade Daniel <edacih@gmail.com>

Philipp Moritz committed Feb 26, 2024 at 13:48 UTC cfc15a1031ef0197a1b291d2ed93717a9bdad268
7 files changed +297 -15
benchmarks/kernels/benchmark_mixtral_moe.py new
+172
@@ -0,0 +1,172 @@
1 +import json
2 +import os
3 +import sys
4 +
5 +os.environ['CUDA_VISIBLE_DEVICES'] = '0'
6 +
7 +from vllm.model_executor.layers.fused_moe import fused_moe
8 +import torch
9 +import torch.nn.functional as F
10 +import triton
11 +
12 +
13 +def main():
14 + method = fused_moe
15 + for bs in [
16 + 1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
17 + 2048, 3072, 4096
18 + ]:
19 + run_grid(bs, method=method)
20 +
21 +
22 +def run_grid(bs, method):
23 + d_model = 4096
24 + num_total_experts = 8
25 + top_k = 2
26 + tp_size = 2
27 + model_intermediate_size = 14336
28 + num_layers = 32
29 + num_calls = 100
30 +
31 + num_warmup_trials = 1
32 + num_trials = 1
33 +
34 + configs = []
35 + if bs <= 16:
36 + BLOCK_SIZES_M = [16]
37 + elif bs <= 32:
38 + BLOCK_SIZES_M = [16, 32]
39 + elif bs <= 64:
40 + BLOCK_SIZES_M = [16, 32, 64]
41 + elif bs <= 128:
42 + BLOCK_SIZES_M = [16, 32, 64, 128]
43 + else:
44 + BLOCK_SIZES_M = [16, 32, 64, 128, 256]
45 +
46 + for block_size_n in [32, 64, 128, 256]:
47 + for block_size_m in BLOCK_SIZES_M:
48 + for block_size_k in [64, 128, 256]:
49 + for group_size_m in [1, 16, 32, 64]:
50 + for num_warps in [4, 8]:
51 + configs.append({
52 + "BLOCK_SIZE_M": block_size_m,
53 + "BLOCK_SIZE_N": block_size_n,
54 + "BLOCK_SIZE_K": block_size_k,
55 + "GROUP_SIZE_M": group_size_m,
56 + "num_warps": num_warps,
57 + "num_stages": 4,
58 + })
59 +
60 + best_config = None
61 + best_time_us = 1e20
62 +
63 + for config in configs:
64 + print(f'{tp_size=} {bs=}')
65 + print(f'{config}')
66 + # warmup
67 + print(f'warming up')
68 + try:
69 + for _ in range(num_warmup_trials):
70 + run_timing(
71 + num_calls=num_calls,
72 + bs=bs,
73 + d_model=d_model,
74 + num_total_experts=num_total_experts,
75 + top_k=top_k,
76 + tp_size=tp_size,
77 + model_intermediate_size=model_intermediate_size,
78 + method=method,
79 + config=config,
80 + )
81 + except triton.runtime.autotuner.OutOfResources:
82 + continue
83 +
84 + # trial
85 + print(f'benchmarking')
86 + for _ in range(num_trials):
87 + kernel_dur_ms = run_timing(
88 + num_calls=num_calls,
89 + bs=bs,
90 + d_model=d_model,
91 + num_total_experts=num_total_experts,
92 + top_k=top_k,
93 + tp_size=tp_size,
94 + model_intermediate_size=model_intermediate_size,
95 + method=method,
96 + config=config,
97 + )
98 +
99 + kernel_dur_us = 1000 * kernel_dur_ms
100 + model_dur_ms = kernel_dur_ms * num_layers
101 +
102 + if kernel_dur_us < best_time_us:
103 + best_config = config
104 + best_time_us = kernel_dur_us
105 +
106 + print(
107 + f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f} {bs=} {tp_size=} {top_k=} {num_total_experts=} {d_model=} {model_intermediate_size=} {num_layers=}'
108 + )
109 +
110 + print("best_time_us", best_time_us)
111 + print("best_config", best_config)
112 +
113 + filename = "/tmp/config.jsonl"
114 + print(f"writing config to file {filename}")
115 + with open(filename, "a") as f:
116 + f.write(json.dumps({str(bs): best_config}) + "\n")
117 +
118 +
119 +def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
120 + top_k: int, tp_size: int, model_intermediate_size: int, method,
121 + config) -> float:
122 + shard_intermediate_size = model_intermediate_size // tp_size
123 +
124 + hidden_states = torch.rand(
125 + (bs, d_model),
126 + device="cuda:0",
127 + dtype=torch.bfloat16,
128 + )
129 +
130 + ws = torch.rand(
131 + (num_total_experts, 2 * shard_intermediate_size, d_model),
132 + device=hidden_states.device,
133 + dtype=hidden_states.dtype,
134 + )
135 +
136 + w2s = torch.rand(
137 + (num_total_experts, d_model, shard_intermediate_size),
138 + device=hidden_states.device,
139 + dtype=hidden_states.dtype,
140 + )
141 +
142 + gating_output = F.softmax(torch.rand(
143 + (num_calls, bs, num_total_experts),
144 + device=hidden_states.device,
145 + dtype=torch.float32,
146 + ),
147 + dim=-1)
148 +
149 + start_event = torch.cuda.Event(enable_timing=True)
150 + end_event = torch.cuda.Event(enable_timing=True)
151 +
152 + start_event.record()
153 + for i in range(num_calls):
154 + hidden_states = method(
155 + hidden_states=hidden_states,
156 + w1=ws,
157 + w2=w2s,
158 + gating_output=gating_output[i],
159 + topk=2,
160 + renormalize=True,
161 + inplace=True,
162 + override_config=config,
163 + )
164 + end_event.record()
165 + end_event.synchronize()
166 +
167 + dur_ms = start_event.elapsed_time(end_event) / num_calls
168 + return dur_ms
169 +
170 +
171 +if __name__ == "__main__":
172 + sys.exit(main())
setup.py
+3 -1
@@ -432,7 +432,9 @@ def get_requirements() -> List[str]:
432 return requirements
433
434
435 -package_data = {"vllm": ["py.typed"]}
435 +package_data = {
436 + "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
437 +}
438 if os.environ.get("VLLM_USE_PRECOMPILED"):
439 ext_modules = []
440 package_data["vllm"].append("*.so")
vllm/model_executor/layers/fused_moe/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
2 +
3 +__all__ = [
4 + "fused_moe",
5 +]
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json new
+20
@@ -0,0 +1,20 @@
1 +{
2 + "1": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
3 + "2": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
4 + "4": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
5 + "8": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
6 + "16": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
7 + "24": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
8 + "32": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
9 + "64": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
10 + "96": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
11 + "128": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
12 + "192": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
13 + "256": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4},
14 + "512": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 32, "num_warps": 8, "num_stages": 4},
15 + "1024": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
16 + "1536": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4},
17 + "2048": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
18 + "3072": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 8, "num_stages": 4},
19 + "4096": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4}
20 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json new
+24
@@ -0,0 +1,24 @@
1 +{
2 + "1": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
3 + "2": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 256, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
4 + "4": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 256, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
5 + "8": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 8, "num_stages": 4},
6 + "16": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
7 + "24": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
8 + "32": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
9 + "80": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
10 + "96": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
11 + "128": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
12 + "192": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
13 + "200": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 4, "num_stages": 4},
14 + "208": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 4, "num_stages": 4},
15 + "216": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
16 + "224": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 32, "num_warps": 4, "num_stages": 4},
17 + "256": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 32, "num_warps": 4, "num_stages": 4},
18 + "512": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
19 + "1024": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
20 + "1536": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
21 + "2048": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
22 + "3072": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
23 + "4096": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4}
24 +}
vllm/model_executor/layers/fused_moe/configs/README new
+10
@@ -0,0 +1,10 @@
1 +This directory contains tuned configurations for different settings of the fused_moe kernel.
2 +For different settings of
3 +- E (number of experts)
4 +- N (intermediate size)
5 +- device_name (torch.cuda.get_device_name())
6 +the JSON file contains a mapping from M (batch size) to the chosen configuration.
7 +
8 +The example configurations provided are for the Mixtral model for TP2 on H100
9 +and TP4 on A100. Mixtral has intermediate size N = 14336, i.e. for TP2 we have
10 +N = 7168 and for TP4 we have N = 3584.
vllm/model_executor/layers/fused_moe/fused_moe.py renamed
+63 -14
@@ -1,11 +1,19 @@
1 """Fused MoE kernel."""
2 +import functools
3 +import json
4 +import os
5 +from typing import Any, Dict, Optional
6 +
7 import torch
8 import triton
9 import triton.language as tl
10
11 from vllm._C import ops
12 +from vllm.logger import init_logger
13 from vllm.utils import is_hip
14
15 +logger = init_logger(__name__)
16 +
17
18 @triton.jit
19 def fused_moe_kernel(
@@ -210,6 +218,34 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
218 )
219
220
221 +@functools.lru_cache
222 +def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
223 + """
224 + Return optimized configurations for the fused MoE kernel.
225 +
226 + The return value will be a dictionary that maps an irregular grid of batch sizes
227 + to configurations of the fused_moe kernel. To evaluate the kernel on a given batch
228 + size bs, the closest batch size in the grid should be picked and the associated
229 + configuration chosen to invoke the kernel.
230 + """
231 +
232 + # First look up if an optimized configuration is available in the configs directory
233 + device_name = torch.cuda.get_device_name().replace(" ", "_")
234 +
235 + config_file_path = os.path.join(
236 + os.path.dirname(os.path.realpath(__file__)), "configs",
237 + f"E={E},N={N},device_name={device_name}.json")
238 + if os.path.exists(config_file_path):
239 + with open(config_file_path) as f:
240 + logger.info(
241 + f"Using configuration from {config_file_path} for MoE layer.")
242 + # If a configuration has been found, return it
243 + return {int(key): val for key, val in json.load(f).items()}
244 +
245 + # If no optimized configuration is available, we will use the default configuration
246 + return None
247 +
248 +
249 def fused_moe(
250 hidden_states: torch.Tensor,
251 w1: torch.Tensor,
@@ -218,6 +254,7 @@ def fused_moe(
254 topk: int,
255 renormalize: bool,
256 inplace: bool = False,
257 + override_config: Optional[Dict[str, Any]] = None,
258 ) -> torch.Tensor:
259 """
260 This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
@@ -230,6 +267,7 @@ def fused_moe(
267 - topk (int): The number of top-k experts to select.
268 - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
269 - inplace (bool): If True, perform the operation in-place. Defaults to False.
270 + - override_config (Optional[Dict[str, Any]]): Optional override for the kernel configuration.
271
272 Returns:
273 - torch.Tensor: The output tensor after applying the MoE layer.
@@ -279,20 +317,31 @@ def fused_moe(
317 if renormalize:
318 topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
319
282 - config = {
283 - 'BLOCK_SIZE_M': 64,
284 - 'BLOCK_SIZE_N': 64,
285 - 'BLOCK_SIZE_K': 32,
286 - 'GROUP_SIZE_M': 8
287 - }
288 -
289 - if topk_ids.numel() <= w1.shape[0]:
290 - config = {
291 - 'BLOCK_SIZE_M': 16,
292 - 'BLOCK_SIZE_N': 32,
293 - 'BLOCK_SIZE_K': 64,
294 - 'GROUP_SIZE_M': 1
295 - }
320 + if override_config:
321 + config = override_config
322 + else:
323 + # First try to load optimal config from the file
324 + configs = get_moe_configs(E, w2.shape[2])
325 +
326 + if configs:
327 + # If an optimal configuration map has been found, look up the optimal config
328 + config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
329 + else:
330 + # Else use the default config
331 + config = {
332 + 'BLOCK_SIZE_M': 64,
333 + 'BLOCK_SIZE_N': 64,
334 + 'BLOCK_SIZE_K': 32,
335 + 'GROUP_SIZE_M': 8
336 + }
337 +
338 + if M <= E:
339 + config = {
340 + 'BLOCK_SIZE_M': 16,
341 + 'BLOCK_SIZE_N': 32,
342 + 'BLOCK_SIZE_K': 64,
343 + 'GROUP_SIZE_M': 1
344 + }
345
346 intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
347 device=hidden_states.device,