A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 20:45 UTC
[ModelRunner V2] Don't pin reused flashinfer tensors (#32799)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Jan 21, 2026 at 13:17 UTC
24dc30f7ff4b304088297ffa2b34ab9aba07bea8
1 file changed
+6
-1
vllm/v1/attention/backends/flashinfer.py
+6
-1
@@ -603,7 +603,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
603
"earlier GPUs."
604
)
605
# Preparing persistent buffers
606
- self.pin_memory = is_pin_memory_available()
606
+ # Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
607
+ # reused CPU buffers to avoid a race condition between step N async copies to
608
+ # GPU and step N+1 buffer updates.
609
+ self.pin_memory = (
610
+ not envs.VLLM_USE_V2_MODEL_RUNNER and is_pin_memory_available()
611
+ )
612
self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1)
613
self.paged_kv_indptr_cpu_buffer = torch.zeros_like(
614
self.paged_kv_indptr.cpu, pin_memory=self.pin_memory