A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
Fill TorchSDPAAttentionMetadata seq_lens_field for prefill (#10799)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Dec 1, 2024 at 23:05 UTC
e25810ae29058299b7bf845c7ed572f2474a1d85
1 file changed
+5
-1
vllm/attention/backends/torch_sdpa.py
+5
-1
@@ -341,7 +341,11 @@ class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
341
)
342
else:
343
block_tables = torch.tensor([])
344
- seq_lens_tensor = torch.tensor([])
344
+ seq_lens_tensor = torch.tensor(
345
+ input_data.seq_lens[:input_data.num_prefills],
346
+ dtype=torch.int32,
347
+ device="cpu",
348
+ )
349
350
# For multi-modal models
351
placeholder_index_maps = None