A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Fix wrong dtype in PagedAttentionWithALiBi bias (#996)
--------- Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>
Antoni Baum committed
Sep 9, 2023 at 14:58 UTC
a62de9ecfdc6bfc80db3d0d1136124c74c94ea04
1 file changed
+11
-4
vllm/model_executor/layers/attention.py
+11
-4
@@ -73,7 +73,12 @@ class PagedAttention(nn.Module):
73
raise ValueError(f"head_size ({self.head_size}) is not supported. "
74
f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
75
76
- def set_attn_bias(self, input_metadata: InputMetadata) -> None:
76
+ def set_attn_bias(
77
+ self,
78
+ input_metadata: InputMetadata,
79
+ dtype: torch.dtype,
80
+ ) -> None:
81
+ del dtype # Unused.
82
if input_metadata.attn_bias:
83
# Already set by a previous layer.
84
return
@@ -196,7 +201,7 @@ class PagedAttention(nn.Module):
201
if num_prompt_tokens > 0:
202
# Prompt run.
203
assert input_metadata.num_generation_tokens == 0
199
- self.set_attn_bias(input_metadata)
204
+ self.set_attn_bias(input_metadata, dtype=query.dtype)
205
self.multi_query_kv_attention(
206
output[:num_prompt_tokens],
207
query[:num_prompt_tokens],
@@ -340,13 +345,14 @@ class PagedAttentionWithALiBi(PagedAttention):
345
slopes = torch.tensor(slopes, dtype=torch.float32)
346
self.register_buffer("alibi_slopes", slopes, persistent=False)
347
343
- def set_attn_bias(self, input_metadata: InputMetadata) -> None:
348
+ def set_attn_bias(self, input_metadata: InputMetadata,
349
+ dtype: torch.dtype) -> None:
350
if input_metadata.attn_bias:
351
# Already set by a previous layer.
352
return
353
# Generates ALiBi mask for each prompt.
354
for prompt_len in input_metadata.prompt_lens:
349
- bias = torch.arange(prompt_len)
355
+ bias = torch.arange(prompt_len, dtype=dtype)
356
# Note(zhuohan): HF uses
357
# `bias = bias[None, :].repeat(prompt_len, 1)`
358
# here. We find that both biases give the same results, but
@@ -364,6 +370,7 @@ class PagedAttentionWithALiBi(PagedAttention):
370
prompt_len,
371
padded_len,
372
device=self.alibi_slopes.device,
373
+ dtype=dtype,
374
)[:, :, :, :prompt_len].copy_(bias)
375
bias.mul_(self.alibi_slopes[:, None, None])
376
attn_bias = LowerTriangularMaskWithTensorBias(bias)