A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix] Fix missing seq_start_loc in xformers prefill metadata (#12464)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 27, 2025 at 15:25 UTC
372bf0890b19cc3c2992ce5c16eca3647e2a9e13
1 file changed
+3
vllm/attention/backends/xformers.py
+3
@@ -199,6 +199,8 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
199
# Compute some attn_metadata fields which default to None
200
query_start_loc = (None if self.query_start_loc is None else
201
self.query_start_loc[:self.num_prefills + 1])
202
+ seq_start_loc = (None if self.seq_start_loc is None else
203
+ self.seq_start_loc[:self.num_prefills + 1])
204
slot_mapping = (None if self.slot_mapping is None else
205
self.slot_mapping[:self.num_prefill_tokens])
206
seq_lens = (None if self.seq_lens is None else
@@ -225,6 +227,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
227
max_prefill_seq_len=self.max_prefill_seq_len,
228
max_decode_seq_len=0,
229
query_start_loc=query_start_loc,
230
+ seq_start_loc=seq_start_loc,
231
context_lens_tensor=context_lens_tensor,
232
block_tables=block_tables,
233
use_cuda_graph=False,