@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa32207842
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Kernel] Fix compute_type for MoE kernel (#4463)

Woosuk Kwon committed Apr 29, 2024 at 22:05 UTC fa32207842f1ed5a966372ed0513914bff8426c4
1 file changed +4 -2
vllm/model_executor/layers/fused_moe/fused_moe.py
+4 -2
@@ -433,6 +433,8 @@ def fused_moe(
433
434 sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
435 topk_ids, config['BLOCK_SIZE_M'], E)
436 + compute_type = (tl.bfloat16
437 + if hidden_states.dtype == torch.bfloat16 else tl.float16)
438
439 invoke_fused_moe_kernel(hidden_states,
440 w1,
@@ -447,7 +449,7 @@ def fused_moe(
449 False,
450 topk_ids.shape[1],
451 config,
450 - compute_type=tl.float16,
452 + compute_type=compute_type,
453 use_fp8=use_fp8)
454
455 ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
@@ -465,7 +467,7 @@ def fused_moe(
467 True,
468 1,
469 config,
468 - compute_type=tl.float16,
470 + compute_type=compute_type,
471 use_fp8=use_fp8)
472
473 if inplace: