A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix][Hardware][CPU] Enable Gemma2 with SDPA on CPU backend (#11169)
Jani Monoses committed
Dec 13, 2024 at 20:00 UTC
0a56bcc03de0857be464c3f8783258d590cbc762
1 file changed
+4
-3
vllm/attention/backends/torch_sdpa.py
+4
-3
@@ -13,7 +13,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
from vllm.attention.backends.utils import CommonAttentionState
14
from vllm.attention.ops.ipex_attn import PagedAttention
15
from vllm.attention.ops.paged_attn import PagedAttentionMetadata
16
-from vllm.utils import make_tensor_with_pad
16
+from vllm.utils import make_tensor_with_pad, print_warning_once
17
from vllm.worker.cpu_model_runner import ModelInputForCPUBuilder
18
19
@@ -395,7 +395,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
395
raise ValueError(
396
"Torch SPDA does not support block-sparse attention.")
397
if logits_soft_cap is not None:
398
- raise ValueError("Torch SPDA does not support logits soft cap.")
398
+ print_warning_once("Torch SPDA does not support logits soft cap. "
399
+ "Outputs may be slightly off.")
400
self.num_heads = num_heads
401
self.head_size = head_size
402
self.scale = float(scale)
@@ -619,7 +620,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
620
value[None, :, start_kv:end_kv, :],
621
attn_mask=mask,
622
dropout_p=0.0,
622
- is_causal=causal_attn and not self.need_mask,
623
+ is_causal=causal_attn and mask is None,
624
scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0)
625
output[start_q:end_q, :, :] = sub_out
626
start_q, start_kv = end_q, end_kv