A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Make FlashInferMetadataBuilder non-blocking (#25040)
Signed-off-by: Julien Lin <jullin@nvidia.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
nvjullin committed
Sep 20, 2025 at 04:36 UTC
b1a63d1b3be996babec6411e3abe559796f76ca9
1 file changed
+3
-2
vllm/v1/attention/backends/flashinfer.py
+3
-2
@@ -585,9 +585,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
585
kv_data_type=self.kv_cache_dtype,
586
)
587
else:
588
- attn_metadata.qo_indptr_gpu = qo_indptr_cpu.to(self.device)
588
+ attn_metadata.qo_indptr_gpu = qo_indptr_cpu.to(
589
+ self.device, non_blocking=True)
590
attn_metadata.paged_kv_indptr_gpu = paged_kv_indptr_cpu.to(
590
- self.device)
591
+ self.device, non_blocking=True)
592
593
if num_decodes > 0:
594
pure_decode = num_prefills == 0