@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f9ecbb18bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Allow passing logits_soft_cap for xformers backend (#11252)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 17, 2024 at 16:37 UTC f9ecbb18bf03338a4272c933a49a87021363b048
1 file changed +3 -5
vllm/attention/backends/xformers.py
+3 -5
@@ -17,9 +17,7 @@ from vllm.attention.backends.utils import (
17 is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set)
18 from vllm.attention.ops.paged_attn import (PagedAttention,
19 PagedAttentionMetadata)
20 -from vllm.logger import init_logger
21 -
22 -logger = init_logger(__name__)
20 +from vllm.utils import print_warning_once
21
22
23 class XFormersBackend(AttentionBackend):
@@ -386,8 +384,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
384 raise ValueError(
385 "XFormers does not support block-sparse attention.")
386 if logits_soft_cap is not None:
389 - raise ValueError(
390 - "XFormers does not support attention logits soft capping.")
387 + print_warning_once("XFormers does not support logits soft cap. "
388 + "Outputs may be slightly off.")
389 self.num_heads = num_heads
390 self.head_size = head_size
391 self.scale = float(scale)