A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[attn][tiny fix] fix attn backend in MultiHeadAttention (#11463)
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Mengqing Cao committed
Dec 24, 2024 at 20:39 UTC
5c7963249daf0b57e803605079e8869e8b071247
1 file changed
+1
vllm/attention/layer.py
+1
@@ -191,6 +191,7 @@ class MultiHeadAttention(nn.Module):
191
kv_cache_dtype=None,
192
block_size=16,
193
is_attention_free=False)
194
+ attn_backend = backend_name_to_enum(attn_backend.get_name())
195
if attn_backend in {_Backend.FLASH_ATTN, _Backend.FLASH_ATTN_VLLM_V1}:
196
attn_backend = _Backend.XFORMERS
197