A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[BugFix] [Kernel] Fix GPU SEGV occuring in fused_moe kernel (#10385)
Signed-off-by: Randall Smith <Randall.Smith@amd.com>
rasmith committed
Nov 16, 2024 at 03:55 UTC
1d754726265d52773653e53e1a18f6eb63122480
1 file changed
+5
-3
vllm/model_executor/layers/fused_moe/fused_moe.py
+5
-3
@@ -105,16 +105,18 @@ def fused_moe_kernel(
105
num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
106
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
107
return
108
- offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
108
+ offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(
109
+ tl.int64)
110
offs_token = tl.load(sorted_token_ids_ptr + offs_token_id)
111
token_mask = offs_token < num_valid_tokens
112
112
- offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
113
+ offs_bn = (pid_n * BLOCK_SIZE_N +
114
+ tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
115
offs_k = tl.arange(0, BLOCK_SIZE_K)
116
a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am +
117
offs_k[None, :] * stride_ak)
118
117
- off_experts = tl.load(expert_ids_ptr + pid_m)
119
+ off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
120
b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
121
offs_bn[None, :] * stride_bn)
122
if use_int8_w8a16: