@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 372bf0890b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] Fix missing seq_start_loc in xformers prefill metadata (#12464)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 27, 2025 at 15:25 UTC 372bf0890b19cc3c2992ce5c16eca3647e2a9e13
1 file changed +3
vllm/attention/backends/xformers.py
+3
@@ -199,6 +199,8 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
199 # Compute some attn_metadata fields which default to None
200 query_start_loc = (None if self.query_start_loc is None else
201 self.query_start_loc[:self.num_prefills + 1])
202 + seq_start_loc = (None if self.seq_start_loc is None else
203 + self.seq_start_loc[:self.num_prefills + 1])
204 slot_mapping = (None if self.slot_mapping is None else
205 self.slot_mapping[:self.num_prefill_tokens])
206 seq_lens = (None if self.seq_lens is None else
@@ -225,6 +227,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
227 max_prefill_seq_len=self.max_prefill_seq_len,
228 max_decode_seq_len=0,
229 query_start_loc=query_start_loc,
230 + seq_start_loc=seq_start_loc,
231 context_lens_tensor=context_lens_tensor,
232 block_tables=block_tables,
233 use_cuda_graph=False,