A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Fix illegal memory access in FP8 MoE kernel (#6382)
Cody Yu committed
Jul 12, 2024 at 14:33 UTC
75f64d8b94d012ea37dddde1058ce17e55001a4a
1 file changed
+5
-3
vllm/model_executor/layers/fused_moe/fused_moe.py
+5
-3
@@ -492,12 +492,14 @@ def fused_experts(hidden_states: torch.Tensor,
492
if tokens_in_chunk == 0:
493
break
494
495
- if tokens_in_chunk < CHUNK_SIZE:
496
- # will only happen in the last chunk
495
+ if tokens_in_chunk < CHUNK_SIZE and chunk > 0:
496
+ # Adjust the intermediate cache size and config for the last
497
+ # chunk. Note that in most cases we only have one chunk
498
+ # so the cache size and config are already set correctly and
499
+ # do not need to be adjusted.
500
intermediate_cache1 = intermediate_cache1[:tokens_in_chunk]
501
intermediate_cache2 = intermediate_cache2[:tokens_in_chunk]
502
intermediate_cache3 = intermediate_cache3[:tokens_in_chunk]
500
- # reload config to get better performance on the last chunk
503
config = get_config_func(tokens_in_chunk)
504
505
curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx]