A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix] Allocate less memory in non-batched CUTLASS MoE (#21121)
Signed-off-by: ElizaWszola <ewszola@redhat.com>
ElizaWszola committed
Jul 18, 2025 at 12:55 UTC
4adc66f64d56338489d00d94de6e13d95741c4be
1 file changed
+2
-2
vllm/model_executor/layers/fused_moe/cutlass_moe.py
+2
-2
@@ -283,8 +283,8 @@ class CutlassExpertsFp8(mk.FusedMoEPermuteExpertsUnpermute):
283
(N // 2))
284
output = (self.max_experts_per_worker, padded_M, K)
285
else:
286
- workspace1 = (M * topk, max(2 * N, K))
287
- workspace2 = (M * topk, N)
286
+ workspace1 = (M * topk, max(N, K))
287
+ workspace2 = (M * topk, N // 2)
288
output = (M * topk, K)
289
return (workspace1, workspace2, output,
290
self.out_dtype if self.out_dtype is not None else a.dtype)