A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Fix expert_ids shape in MoE (#4517)
Woosuk Kwon committed
May 1, 2024 at 16:47 UTC
826b82a260ebb1ea7edd04a3278d5fb9b103a76e
1 file changed
+6
-5
vllm/model_executor/layers/fused_moe/fused_moe.py
+6
-5
@@ -203,14 +203,15 @@ def moe_align_block_size(
203
- The padding ensures that the total number of tokens is now divisible
204
by block_size for proper block matrix operations.
205
"""
206
- sorted_ids = torch.empty(
207
- (topk_ids.numel() + num_experts * (block_size - 1), ),
208
- dtype=torch.int32,
209
- device=topk_ids.device)
210
- expert_ids = torch.empty((topk_ids.numel() + num_experts, ),
206
+ max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1)
207
+ sorted_ids = torch.empty((max_num_tokens_padded, ),
208
dtype=torch.int32,
209
device=topk_ids.device)
210
sorted_ids.fill_(topk_ids.numel())
211
+ max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
212
+ expert_ids = torch.empty((max_num_m_blocks, ),
213
+ dtype=torch.int32,
214
+ device=topk_ids.device)
215
num_tokens_post_pad = torch.empty((1),
216
dtype=torch.int32,
217
device=topk_ids.device)