A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[full_graph] Fix query_start_loc padding (#19321)
Signed-off-by: Yinghai Lu <yinghai@thinkingmachines.ai>
Yinghai Lu committed
Jun 9, 2025 at 06:32 UTC
770e5dcdb8c2e8bc31171e614efc471d4a0c2442
1 file changed
+4
-1
vllm/v1/worker/gpu_model_runner.py
+4
-1
@@ -655,7 +655,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
655
656
# Fill unused with -1. Needed for reshape_and_cache
657
self.seq_lens[num_reqs:].fill_(0)
658
- self.query_start_loc[num_reqs + 1:].fill_(-1)
658
+ # Note: pad query_start_loc to be non-decreasing, as kernels
659
+ # like FlashAttention requires that
660
+ self.query_start_loc[num_reqs + 1:].fill_(
661
+ self.query_start_loc_cpu[num_reqs].item())
662
663
query_start_loc = self.query_start_loc[:num_reqs + 1]
664
seq_lens = self.seq_lens[:num_reqs]