@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75f64d8b94
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Fix illegal memory access in FP8 MoE kernel (#6382)

Cody Yu committed Jul 12, 2024 at 14:33 UTC 75f64d8b94d012ea37dddde1058ce17e55001a4a
1 file changed +5 -3
vllm/model_executor/layers/fused_moe/fused_moe.py
+5 -3
@@ -492,12 +492,14 @@ def fused_experts(hidden_states: torch.Tensor,
492 if tokens_in_chunk == 0:
493 break
494
495 - if tokens_in_chunk < CHUNK_SIZE:
496 - # will only happen in the last chunk
495 + if tokens_in_chunk < CHUNK_SIZE and chunk > 0:
496 + # Adjust the intermediate cache size and config for the last
497 + # chunk. Note that in most cases we only have one chunk
498 + # so the cache size and config are already set correctly and
499 + # do not need to be adjusted.
500 intermediate_cache1 = intermediate_cache1[:tokens_in_chunk]
501 intermediate_cache2 = intermediate_cache2[:tokens_in_chunk]
502 intermediate_cache3 = intermediate_cache3[:tokens_in_chunk]
500 - # reload config to get better performance on the last chunk
503 config = get_config_func(tokens_in_chunk)
504
505 curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx]