A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
Fix precommit fail in fused_moe intermediate_cache2 chunking (#13772)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Feb 24, 2025 at 12:25 UTC
db986c19ea35d7f3522a45d5205bf5d3ffab14e4
1 file changed
+2
-1
vllm/model_executor/layers/fused_moe/fused_moe.py
+2
-1
@@ -1271,7 +1271,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1271
# so the cache size and config are already set correctly and
1272
# do not need to be adjusted.
1273
intermediate_cache1 = intermediate_cache1[:tokens_in_chunk]
1274
- intermediate_cache2 = intermediate_cache2[:tokens_in_chunk * topk_ids.shape[1]]
1274
+ intermediate_cache2 = intermediate_cache2[:tokens_in_chunk *
1275
+ topk_ids.shape[1]]
1276
intermediate_cache3 = intermediate_cache3[:tokens_in_chunk]
1277
config = get_config_func(tokens_in_chunk)
1278