@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d75472626
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[BugFix] [Kernel] Fix GPU SEGV occuring in fused_moe kernel (#10385)

Signed-off-by: Randall Smith <Randall.Smith@amd.com>

rasmith committed Nov 16, 2024 at 03:55 UTC 1d754726265d52773653e53e1a18f6eb63122480
1 file changed +5 -3
vllm/model_executor/layers/fused_moe/fused_moe.py
+5 -3
@@ -105,16 +105,18 @@ def fused_moe_kernel(
105 num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
106 if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
107 return
108 - offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
108 + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(
109 + tl.int64)
110 offs_token = tl.load(sorted_token_ids_ptr + offs_token_id)
111 token_mask = offs_token < num_valid_tokens
112
112 - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
113 + offs_bn = (pid_n * BLOCK_SIZE_N +
114 + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
115 offs_k = tl.arange(0, BLOCK_SIZE_K)
116 a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am +
117 offs_k[None, :] * stride_ak)
118
117 - off_experts = tl.load(expert_ids_ptr + pid_m)
119 + off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
120 b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
121 offs_bn[None, :] * stride_bn)
122 if use_int8_w8a16: