A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 16:30 UTC
[Bugfix]: Fix cross attention backend selection for Turing GPU (#31806)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jan 6, 2026 at 23:15 UTC
02809af1e7cdf63bcf0d73b63a35ad3ea5c6fb81
1 file changed
+9
-5
vllm/attention/layers/cross_attention.py
+9
-5
@@ -149,16 +149,20 @@ class CrossAttention(Attention):
149
kv_cache_dtype = "auto"
150
block_size = 16
151
152
- underlying_attn_backend = get_attn_backend(
153
- head_size, dtype, kv_cache_dtype, block_size
154
- )
155
- attn_backend = create_cross_attention_backend(underlying_attn_backend)
156
-
152
if attn_type is not None:
153
assert attn_type == AttentionType.ENCODER_DECODER, (
154
"CrossAttention only supports AttentionType.ENCODER_DECODER"
155
)
156
157
+ underlying_attn_backend = get_attn_backend(
158
+ head_size,
159
+ dtype,
160
+ kv_cache_dtype,
161
+ block_size,
162
+ attn_type=AttentionType.ENCODER_DECODER,
163
+ )
164
+ attn_backend = create_cross_attention_backend(underlying_attn_backend)
165
+
166
super().__init__(
167
num_heads=num_heads,
168
head_size=head_size,