A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix MTP+FlashInfer crash when trtllm kernels are available but disabled (#26361)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Benjamin Chislett committed
Oct 7, 2025 at 18:12 UTC
caf8b1c0840b804bf36db30e57f14e5ecf3126dd
1 file changed
+2
vllm/utils/flashinfer.py
+2
@@ -220,6 +220,8 @@ def force_use_trtllm_attention() -> bool | None:
220
221
def can_use_trtllm_attention(num_qo_heads: int, num_kv_heads: int) -> bool:
222
"""Check if the current configuration supports TRTLLM attention."""
223
+ if force_use_trtllm_attention() is False:
224
+ return False
225
has_trtllm = supports_trtllm_attention()
226
return has_trtllm and (num_qo_heads % num_kv_heads == 0)
227