A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Kernels] Use empty for modular MoE workspaces (#19667)
Signed-off-by: Bill Nell <bnell@redhat.com>
bnellnm committed
Jun 16, 2025 at 10:58 UTC
5e5baa91aa163ec3c49ed8b3a49b4dab3430a436
2 files changed
+5
-2
vllm/model_executor/layers/fused_moe/fused_batched_moe.py
+3
@@ -716,6 +716,9 @@ class BatchedTritonExperts(mk.FusedMoEPermuteExpertsUnpermute):
716
intermediate_cache2 = _resize_cache(workspace2,
717
(E, max_num_tokens, N // 2))
718
719
+ if self.use_fp8_w8a8:
720
+ intermediate_cache1.fill_(0)
721
+
722
# MM1
723
invoke_moe_batched_triton_kernel(A=hidden_states,
724
B=w1,
vllm/model_executor/layers/fused_moe/modular_kernel.py
+2
-2
@@ -426,10 +426,10 @@ class FusedMoEModularKernel(torch.nn.Module):
426
427
# We can reuse the memory between cache1 and cache3 because by the
428
# time we need cache3, we're done with cache1.
429
- workspace13 = torch.zeros(prod(workspace13_shape),
429
+ workspace13 = torch.empty(prod(workspace13_shape),
430
device=a1.device,
431
dtype=workspace_dtype)
432
- workspace2 = torch.zeros(prod(workspace2_shape),
432
+ workspace2 = torch.empty(prod(workspace2_shape),
433
device=a1.device,
434
dtype=workspace_dtype)
435