@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a62de9ecfd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Fix wrong dtype in PagedAttentionWithALiBi bias (#996)

--------- Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>

Antoni Baum committed Sep 9, 2023 at 14:58 UTC a62de9ecfdc6bfc80db3d0d1136124c74c94ea04
1 file changed +11 -4
vllm/model_executor/layers/attention.py
+11 -4
@@ -73,7 +73,12 @@ class PagedAttention(nn.Module):
73 raise ValueError(f"head_size ({self.head_size}) is not supported. "
74 f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
75
76 - def set_attn_bias(self, input_metadata: InputMetadata) -> None:
76 + def set_attn_bias(
77 + self,
78 + input_metadata: InputMetadata,
79 + dtype: torch.dtype,
80 + ) -> None:
81 + del dtype # Unused.
82 if input_metadata.attn_bias:
83 # Already set by a previous layer.
84 return
@@ -196,7 +201,7 @@ class PagedAttention(nn.Module):
201 if num_prompt_tokens > 0:
202 # Prompt run.
203 assert input_metadata.num_generation_tokens == 0
199 - self.set_attn_bias(input_metadata)
204 + self.set_attn_bias(input_metadata, dtype=query.dtype)
205 self.multi_query_kv_attention(
206 output[:num_prompt_tokens],
207 query[:num_prompt_tokens],
@@ -340,13 +345,14 @@ class PagedAttentionWithALiBi(PagedAttention):
345 slopes = torch.tensor(slopes, dtype=torch.float32)
346 self.register_buffer("alibi_slopes", slopes, persistent=False)
347
343 - def set_attn_bias(self, input_metadata: InputMetadata) -> None:
348 + def set_attn_bias(self, input_metadata: InputMetadata,
349 + dtype: torch.dtype) -> None:
350 if input_metadata.attn_bias:
351 # Already set by a previous layer.
352 return
353 # Generates ALiBi mask for each prompt.
354 for prompt_len in input_metadata.prompt_lens:
349 - bias = torch.arange(prompt_len)
355 + bias = torch.arange(prompt_len, dtype=dtype)
356 # Note(zhuohan): HF uses
357 # `bias = bias[None, :].repeat(prompt_len, 1)`
358 # here. We find that both biases give the same results, but
@@ -364,6 +370,7 @@ class PagedAttentionWithALiBi(PagedAttention):
370 prompt_len,
371 padded_len,
372 device=self.alibi_slopes.device,
373 + dtype=dtype,
374 )[:, :, :, :prompt_len].copy_(bias)
375 bias.mul_(self.alibi_slopes[:, None, None])
376 attn_bias = LowerTriangularMaskWithTensorBias(bias)