@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a56bcc03d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix][Hardware][CPU] Enable Gemma2 with SDPA on CPU backend (#11169)

Jani Monoses committed Dec 13, 2024 at 20:00 UTC 0a56bcc03de0857be464c3f8783258d590cbc762
1 file changed +4 -3
vllm/attention/backends/torch_sdpa.py
+4 -3
@@ -13,7 +13,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 from vllm.attention.backends.utils import CommonAttentionState
14 from vllm.attention.ops.ipex_attn import PagedAttention
15 from vllm.attention.ops.paged_attn import PagedAttentionMetadata
16 -from vllm.utils import make_tensor_with_pad
16 +from vllm.utils import make_tensor_with_pad, print_warning_once
17 from vllm.worker.cpu_model_runner import ModelInputForCPUBuilder
18
19
@@ -395,7 +395,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
395 raise ValueError(
396 "Torch SPDA does not support block-sparse attention.")
397 if logits_soft_cap is not None:
398 - raise ValueError("Torch SPDA does not support logits soft cap.")
398 + print_warning_once("Torch SPDA does not support logits soft cap. "
399 + "Outputs may be slightly off.")
400 self.num_heads = num_heads
401 self.head_size = head_size
402 self.scale = float(scale)
@@ -619,7 +620,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
620 value[None, :, start_kv:end_kv, :],
621 attn_mask=mask,
622 dropout_p=0.0,
622 - is_causal=causal_attn and not self.need_mask,
623 + is_causal=causal_attn and mask is None,
624 scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0)
625 output[start_q:end_q, :, :] = sub_out
626 start_q, start_kv = end_q, end_kv