21
a_ptr,
22
b_ptr,
23
c_ptr,
24
+ a_scale_ptr,
25
+ b_scale_ptr,
26
topk_weights_ptr,
27
sorted_token_ids_ptr,
28
expert_ids_ptr,
51
MUL_ROUTED_WEIGHT: tl.constexpr,
52
top_k: tl.constexpr,
53
compute_type: tl.constexpr,
54
+ use_fp8: tl.constexpr,
55
):
56
"""
57
Implements the fused computation for a Mixture of Experts (MOE) using
114
b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
115
offs_bn[None, :] * stride_bn)
116
117
+ if use_fp8:
118
+ a_scale = tl.load(a_scale_ptr)
119
+ b_scale = tl.load(b_scale_ptr + off_experts)
120
+
121
# -----------------------------------------------------------
122
# Iterate to compute a block of the C matrix.
123
# We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
136
mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
137
other=0.0)
138
# We accumulate along the K dimension.
132
- accumulator += tl.dot(a, b)
139
+ if use_fp8:
140
+ accumulator = tl.dot(a, b, acc=accumulator)
141
+ else:
142
+ accumulator += tl.dot(a, b)
143
# Advance the ptrs to the next K block.
144
a_ptrs += BLOCK_SIZE_K * stride_ak
145
b_ptrs += BLOCK_SIZE_K * stride_bk
150
other=0)
151
accumulator = accumulator * moe_weight[:, None]
152
143
- accumulator = accumulator.to(compute_type)
153
+ if use_fp8:
154
+ accumulator = (accumulator * a_scale * b_scale).to(compute_type)
155
+ else:
156
+ accumulator = accumulator.to(compute_type)
157
# -----------------------------------------------------------
158
# Write back the block of the output
159
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
220
221
222
def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
210
- topk_weights: torch.Tensor, topk_ids: torch.Tensor,
223
+ B_scale: torch.Tensor, topk_weights: torch.Tensor,
224
+ topk_ids: torch.Tensor,
225
sorted_token_ids: torch.Tensor,
226
expert_ids: torch.Tensor,
227
num_tokens_post_padded: torch.Tensor,
228
mul_routed_weight: bool, top_k: int,
215
- config: Dict[str, Any]) -> None:
229
+ config: Dict[str, Any], compute_type: tl.dtype,
230
+ use_fp8: bool) -> None:
231
assert topk_weights.stride(1) == 1
232
assert sorted_token_ids.stride(0) == 1
233
234
+ if not use_fp8:
235
+ A_scale = None
236
+ assert B_scale is None
237
+ else:
238
+ A, A_scale = ops.scaled_fp8_quant(A)
239
+ assert B_scale is not None
240
+
241
grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
242
'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
243
245
A,
246
B,
247
C,
248
+ A_scale,
249
+ B_scale,
250
topk_weights,
251
sorted_token_ids,
252
expert_ids,
264
C.stride(2),
265
MUL_ROUTED_WEIGHT=mul_routed_weight,
266
top_k=top_k,
243
- compute_type=tl.bfloat16 if A.dtype == torch.bfloat16 else tl.float16,
267
+ compute_type=compute_type,
268
+ use_fp8=use_fp8,
269
**config,
270
)
271
272
248
-def get_config_file_name(E: int, N: int) -> str:
273
+def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
274
device_name = torch.cuda.get_device_name().replace(" ", "_")
250
- return f"E={E},N={N},device_name={device_name}.json"
275
+ dtype_selector = "" if not dtype else f",dtype={dtype}"
276
+ return f"E={E},N={N},device_name={device_name}{dtype_selector}.json"
277
278
279
@functools.lru_cache
254
-def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
280
+def get_moe_configs(E: int, N: int,
281
+ dtype: Optional[str]) -> Optional[Dict[int, Any]]:
282
"""
283
Return optimized configurations for the fused MoE kernel.
284
290
291
# First look up if an optimized configuration is available in the configs
292
# directory
266
- json_file_name = get_config_file_name(E, N)
293
+ json_file_name = get_config_file_name(E, N, dtype)
294
295
config_file_path = os.path.join(
296
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
315
renormalize: bool,
316
inplace: bool = False,
317
override_config: Optional[Dict[str, Any]] = None,
318
+ use_fp8: bool = False,
319
+ w1_scale: Optional[torch.Tensor] = None,
320
+ w2_scale: Optional[torch.Tensor] = None,
321
) -> torch.Tensor:
322
"""
323
This function computes a Mixture of Experts (MoE) layer using two sets of
335
Defaults to False.
336
- override_config (Optional[Dict[str, Any]]): Optional override
337
for the kernel configuration.
338
+ - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
339
+ products for w1 and w2. Defaults to False.
340
+ - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
341
+ w1.
342
+ - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
343
+ w2.
344
345
Returns:
346
- torch.Tensor: The output tensor after applying the MoE layer.
394
config = override_config
395
else:
396
# First try to load optimal config from the file
361
- configs = get_moe_configs(E, w2.shape[2])
397
+ configs = get_moe_configs(E, w2.shape[2],
398
+ "float8" if use_fp8 else None)
399
400
if configs:
401
# If an optimal configuration map has been found, look up the
431
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
432
topk_ids, config['BLOCK_SIZE_M'], E)
433
397
- invoke_fused_moe_kernel(hidden_states, w1, intermediate_cache1,
398
- topk_weights, topk_ids, sorted_token_ids,
399
- expert_ids, num_tokens_post_padded, False,
400
- topk_ids.shape[1], config)
434
+ invoke_fused_moe_kernel(hidden_states,
435
+ w1,
436
+ intermediate_cache1,
437
+ w1_scale,
438
+ topk_weights,
439
+ topk_ids,
440
+ sorted_token_ids,
441
+ expert_ids,
442
+ num_tokens_post_padded,
443
+ False,
444
+ topk_ids.shape[1],
445
+ config,
446
+ compute_type=tl.float16,
447
+ use_fp8=use_fp8)
448
449
ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
450
404
- invoke_fused_moe_kernel(intermediate_cache2, w2, intermediate_cache3,
405
- topk_weights, topk_ids, sorted_token_ids,
406
- expert_ids, num_tokens_post_padded, True, 1,
407
- config)
451
+ invoke_fused_moe_kernel(intermediate_cache2,
452
+ w2,
453
+ intermediate_cache3,
454
+ w2_scale,
455
+ topk_weights,
456
+ topk_ids,
457
+ sorted_token_ids,
458
+ expert_ids,
459
+ num_tokens_post_padded,
460
+ True,
461
+ 1,
462
+ config,
463
+ compute_type=tl.float16,
464
+ use_fp8=use_fp8)
465
466
if inplace:
467
return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),