@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af8486de49
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Hardware][Intel-Gaudi] Enable FusedSDPA support for Intel Gaudi (HPU)

Sanju C Sudhakaran committed Feb 6, 2025 at 02:59 UTC af8486de49a200a980f71fddc6d1eb4d8f9f1bca
1 file changed +11 -1
vllm/attention/backends/hpu_attn.py
+11 -1
@@ -10,7 +10,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
10
11 import torch
12 import vllm_hpu_extension.ops as ops
13 -from vllm_hpu_extension.utils import Matmul, Softmax, VLLMKVCache
13 +from vllm_hpu_extension.utils import (Matmul, ModuleFusedSDPA, Softmax,
14 + VLLMKVCache)
15
16 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
17 AttentionLayer,
@@ -137,9 +138,17 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
138
139 self.prefill_usefusedsdpa = os.getenv('VLLM_PROMPT_USE_FUSEDSDPA',
140 '0').lower() in ['1', 'true']
141 + self.fused_scaled_dot_product_attention = None
142 if self.prefill_usefusedsdpa:
143 assert alibi_slopes is None, \
144 'Prefill with FusedSDPA not supported with alibi slopes!'
145 + try:
146 + from habana_frameworks.torch.hpex.kernels import FusedSDPA
147 + self.fused_scaled_dot_product_attention = ModuleFusedSDPA(
148 + FusedSDPA)
149 + except ImportError:
150 + logger().warning("Could not import HPU FusedSDPA kernel. "
151 + "vLLM will use native implementation.")
152
153 suppored_head_sizes = HPUPagedAttention.get_supported_head_sizes()
154 if head_size not in suppored_head_sizes:
@@ -227,6 +236,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
236 matmul_qk_op=self.matmul_qk,
237 softmax_op=self.softmax,
238 matmul_av_op=self.matmul_av,
239 + fsdpa_op=self.fused_scaled_dot_product_attention,
240 )
241 output = out.reshape(batch_size, seq_len, hidden_size)
242 else: