@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 826b82a260
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Fix expert_ids shape in MoE (#4517)

Woosuk Kwon committed May 1, 2024 at 16:47 UTC 826b82a260ebb1ea7edd04a3278d5fb9b103a76e
1 file changed +6 -5
vllm/model_executor/layers/fused_moe/fused_moe.py
+6 -5
@@ -203,14 +203,15 @@ def moe_align_block_size(
203 - The padding ensures that the total number of tokens is now divisible
204 by block_size for proper block matrix operations.
205 """
206 - sorted_ids = torch.empty(
207 - (topk_ids.numel() + num_experts * (block_size - 1), ),
208 - dtype=torch.int32,
209 - device=topk_ids.device)
210 - expert_ids = torch.empty((topk_ids.numel() + num_experts, ),
206 + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
207 + sorted_ids = torch.empty((max_num_tokens_padded, ),
208 dtype=torch.int32,
209 device=topk_ids.device)
210 sorted_ids.fill_(topk_ids.numel())
211 + max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
212 + expert_ids = torch.empty((max_num_m_blocks, ),
213 + dtype=torch.int32,
214 + device=topk_ids.device)
215 num_tokens_post_pad = torch.empty((1),
216 dtype=torch.int32,
217 device=topk_ids.device)