A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Allow passing logits_soft_cap for xformers backend (#11252)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 17, 2024 at 16:37 UTC
f9ecbb18bf03338a4272c933a49a87021363b048
1 file changed
+3
-5
vllm/attention/backends/xformers.py
+3
-5
@@ -17,9 +17,7 @@ from vllm.attention.backends.utils import (
17
is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set)
18
from vllm.attention.ops.paged_attn import (PagedAttention,
19
PagedAttentionMetadata)
20
-from vllm.logger import init_logger
21
-
22
-logger = init_logger(__name__)
20
+from vllm.utils import print_warning_once
21
22
23
class XFormersBackend(AttentionBackend):
@@ -386,8 +384,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
384
raise ValueError(
385
"XFormers does not support block-sparse attention.")
386
if logits_soft_cap is not None:
389
- raise ValueError(
390
- "XFormers does not support attention logits soft capping.")
387
+ print_warning_once("XFormers does not support logits soft cap. "
388
+ "Outputs may be slightly off.")
389
self.num_heads = num_heads
390
self.head_size = head_size
391
self.scale = float(scale)