A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix][Kernel] Fix compute_type for MoE kernel (#4463)
Woosuk Kwon committed
Apr 29, 2024 at 22:05 UTC
fa32207842f1ed5a966372ed0513914bff8426c4
1 file changed
+4
-2
vllm/model_executor/layers/fused_moe/fused_moe.py
+4
-2
@@ -433,6 +433,8 @@ def fused_moe(
433
434
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
435
topk_ids, config['BLOCK_SIZE_M'], E)
436
+ compute_type = (tl.bfloat16
437
+ if hidden_states.dtype == torch.bfloat16 else tl.float16)
438
439
invoke_fused_moe_kernel(hidden_states,
440
w1,
@@ -447,7 +449,7 @@ def fused_moe(
449
False,
450
topk_ids.shape[1],
451
config,
450
- compute_type=tl.float16,
452
+ compute_type=compute_type,
453
use_fp8=use_fp8)
454
455
ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
@@ -465,7 +467,7 @@ def fused_moe(
467
True,
468
1,
469
config,
468
- compute_type=tl.float16,
470
+ compute_type=compute_type,
471
use_fp8=use_fp8)
472
473
if inplace: