A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[BugFix] Illegal memory access for MoE On H20 (#13693)
Zhonghua Deng committed
Feb 24, 2025 at 23:37 UTC
ccc00515fde6954a617aea98a927b751d8082946
1 file changed
+1
-1
vllm/model_executor/layers/fused_moe/fused_moe.py
+1
-1
@@ -1271,7 +1271,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1271
# so the cache size and config are already set correctly and
1272
# do not need to be adjusted.
1273
intermediate_cache1 = intermediate_cache1[:tokens_in_chunk]
1274
- intermediate_cache2 = intermediate_cache2[:tokens_in_chunk]
1274
+ intermediate_cache2 = intermediate_cache2[:tokens_in_chunk * topk_ids.shape[1]]
1275
intermediate_cache3 = intermediate_cache3[:tokens_in_chunk]
1276
config = get_config_func(tokens_in_chunk)
1277