A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Hardware][Intel-Gaudi] Enable FusedSDPA support for Intel Gaudi (HPU)
Sanju C Sudhakaran committed
Feb 6, 2025 at 02:59 UTC
af8486de49a200a980f71fddc6d1eb4d8f9f1bca
1 file changed
+11
-1
vllm/attention/backends/hpu_attn.py
+11
-1
@@ -10,7 +10,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
10
11
import torch
12
import vllm_hpu_extension.ops as ops
13
-from vllm_hpu_extension.utils import Matmul, Softmax, VLLMKVCache
13
+from vllm_hpu_extension.utils import (Matmul, ModuleFusedSDPA, Softmax,
14
+ VLLMKVCache)
15
16
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
17
AttentionLayer,
@@ -137,9 +138,17 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
138
139
self.prefill_usefusedsdpa = os.getenv('VLLM_PROMPT_USE_FUSEDSDPA',
140
'0').lower() in ['1', 'true']
141
+ self.fused_scaled_dot_product_attention = None
142
if self.prefill_usefusedsdpa:
143
assert alibi_slopes is None, \
144
'Prefill with FusedSDPA not supported with alibi slopes!'
145
+ try:
146
+ from habana_frameworks.torch.hpex.kernels import FusedSDPA
147
+ self.fused_scaled_dot_product_attention = ModuleFusedSDPA(
148
+ FusedSDPA)
149
+ except ImportError:
150
+ logger().warning("Could not import HPU FusedSDPA kernel. "
151
+ "vLLM will use native implementation.")
152
153
suppored_head_sizes = HPUPagedAttention.get_supported_head_sizes()
154
if head_size not in suppored_head_sizes:
@@ -227,6 +236,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
236
matmul_qk_op=self.matmul_qk,
237
softmax_op=self.softmax,
238
matmul_av_op=self.matmul_av,
239
+ fsdpa_op=self.fused_scaled_dot_product_attention,
240
)
241
output = out.reshape(batch_size, seq_len, hidden_size)
242
else: