A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
Add warning for Attention backends that do not support irope yet (#16212)
Yong Hoon Shin committed
Apr 7, 2025 at 20:59 UTC
05a015d6a52e6093f1ac0b76ada5b7da4d6a5671
8 files changed
+52
vllm/attention/backends/flashinfer.py
+8
@@ -38,9 +38,12 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
38
from vllm.attention.layer import Attention
39
from vllm.attention.ops.paged_attn import PagedAttention
40
from vllm.config import VllmConfig, get_current_vllm_config
41
+from vllm.logger import init_logger
42
from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
43
make_tensor_with_pad)
44
45
+logger = init_logger(__name__)
46
+
47
if TYPE_CHECKING:
48
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
49
ModelInputForGPUWithSamplingMetadata)
@@ -907,7 +910,12 @@ class FlashInferImpl(AttentionImpl):
910
blocksparse_params: Optional[Dict[str, Any]] = None,
911
logits_soft_cap: Optional[float] = None,
912
attn_type: str = AttentionType.DECODER,
913
+ use_irope: bool = False,
914
) -> None:
915
+ if use_irope:
916
+ logger.warning_once(
917
+ "Using irope in FlashInfer is not supported yet, it will fall"
918
+ " back to global attention for long context.")
919
self.num_heads = num_heads
920
self.head_size = head_size
921
self.scale = float(scale)
vllm/attention/backends/hpu_attn.py
+5
@@ -108,8 +108,13 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
108
blocksparse_params: Optional[Dict[str, Any]] = None,
109
max_seq_len: int = 4096,
110
attn_type: str = AttentionType.DECODER,
111
+ use_irope: bool = False,
112
) -> None:
113
super(AttentionImpl, self).__init__()
114
+ if use_irope:
115
+ logger.warning_once(
116
+ "Using irope in HPU is not supported yet, it will fall back "
117
+ "to global attention for long context.")
118
self.kv_cache_dtype = kv_cache_dtype
119
self.num_heads = num_heads
120
self.head_size = head_size
vllm/attention/backends/ipex_attn.py
+8
@@ -14,6 +14,9 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
from vllm.attention.backends.utils import CommonAttentionState
15
from vllm.attention.ops.paged_attn import (PagedAttention,
16
PagedAttentionMetadata)
17
+from vllm.logger import init_logger
18
+
19
+logger = init_logger(__name__)
20
21
_PARTITION_SIZE = 512
22
@@ -119,7 +122,12 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
122
blocksparse_params: Optional[Dict[str, Any]] = None,
123
logits_soft_cap: Optional[float] = None,
124
attn_type: str = AttentionType.DECODER,
125
+ use_irope: bool = False,
126
) -> None:
127
+ if use_irope:
128
+ logger.warning_once(
129
+ "Using irope in Ipex is not supported yet, it will fall"
130
+ " back to global attention for long context.")
131
if blocksparse_params is not None:
132
raise ValueError(
133
"IPEX backend does not support block-sparse attention.")
vllm/attention/backends/pallas.py
+8
@@ -11,6 +11,9 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11
AttentionMetadata, AttentionType,
12
is_quantized_kv_cache)
13
from vllm.attention.backends.utils import CommonAttentionState
14
+from vllm.logger import init_logger
15
+
16
+logger = init_logger(__name__)
17
18
19
class PallasAttentionBackend(AttentionBackend):
@@ -105,7 +108,12 @@ class PallasAttentionBackendImpl(AttentionImpl):
108
blocksparse_params: Optional[Dict[str, Any]] = None,
109
logits_soft_cap: Optional[float] = None,
110
attn_type: str = AttentionType.DECODER,
111
+ use_irope: bool = False,
112
) -> None:
113
+ if use_irope:
114
+ logger.warning_once(
115
+ "Using irope in Pallas is not supported yet, it will fall back "
116
+ "to global attention for long context.")
117
self.num_heads = num_heads
118
self.head_size = head_size
119
self.scale = float(scale)
vllm/attention/backends/rocm_flash_attn.py
+5
@@ -462,7 +462,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
462
blocksparse_params: Optional[Dict[str, Any]] = None,
463
logits_soft_cap: Optional[float] = None,
464
attn_type: str = AttentionType.DECODER,
465
+ use_irope: bool = False,
466
) -> None:
467
+ if use_irope:
468
+ logger.warning_once(
469
+ "Using irope in ROCm Flash Attention is not supported yet, it "
470
+ "will fail back to global attention for long context.")
471
if blocksparse_params is not None:
472
raise ValueError(
473
"ROCmFlashAttention does not support blocksparse attention.")
vllm/attention/backends/torch_sdpa.py
+5
@@ -404,6 +404,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
404
blocksparse_params: Optional[Dict[str, Any]] = None,
405
logits_soft_cap: Optional[float] = None,
406
attn_type: str = AttentionType.DECODER,
407
+ use_irope: bool = False,
408
) -> None:
409
if blocksparse_params is not None:
410
raise ValueError(
@@ -411,6 +412,10 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
412
if logits_soft_cap is not None:
413
logger.warning_once("Torch SPDA does not support logits soft cap. "
414
"Outputs may be slightly off.")
415
+ if use_irope:
416
+ logger.warning_once(
417
+ "Using irope in Torch SPDA is not supported yet, it will fall"
418
+ " back to global attention for long context.")
419
self.num_heads = num_heads
420
self.head_size = head_size
421
self.scale = float(scale)
vllm/attention/backends/xformers.py
+5
@@ -389,6 +389,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
389
blocksparse_params: Optional[Dict[str, Any]] = None,
390
logits_soft_cap: Optional[float] = None,
391
attn_type: str = AttentionType.DECODER,
392
+ use_irope: bool = False,
393
) -> None:
394
if blocksparse_params is not None:
395
raise ValueError(
@@ -396,6 +397,10 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
397
if logits_soft_cap is not None:
398
logger.warning_once("XFormers does not support logits soft cap. "
399
"Outputs may be slightly off.")
400
+ if use_irope:
401
+ logger.warning_once(
402
+ "Using irope in XFormers is not supported yet, it will fall"
403
+ " back to global attention for long context.")
404
self.num_heads = num_heads
405
self.head_size = head_size
406
self.scale = float(scale)
vllm/v1/attention/backends/pallas.py
+8
@@ -10,6 +10,9 @@ import torch_xla.experimental.custom_kernel # noqa: F401
10
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11
AttentionLayer, AttentionType)
12
from vllm.attention.backends.utils import CommonAttentionState
13
+from vllm.logger import init_logger
14
+
15
+logger = init_logger(__name__)
16
17
18
class PallasAttentionBackend(AttentionBackend):
@@ -80,7 +83,12 @@ class PallasAttentionBackendImpl(AttentionImpl):
83
blocksparse_params: Optional[dict[str, Any]] = None,
84
logits_soft_cap: Optional[float] = None,
85
attn_type: str = AttentionType.DECODER,
86
+ use_irope: bool = False,
87
) -> None:
88
+ if use_irope:
89
+ logger.warning_once(
90
+ "Using irope in Pallas is not supported yet, it will fall back "
91
+ "to global attention for long context.")
92
if blocksparse_params is not None:
93
raise ValueError("Paged attention Pallas kernel does "
94
"not support block-sparse attention.")