A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 01:15 UTC
[Perf] Remove blocking copy in GDN Attention (#31167)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Benjamin Chislett committed
Dec 22, 2025 at 17:25 UTC
85aff45e24de7af96d30baa1d7d0fc7aec43c28a
1 file changed
+1
-1
vllm/v1/attention/backends/gdn_attn.py
+1
-1
@@ -143,7 +143,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
143
144
query_start_loc = m.query_start_loc
145
context_lens = m.num_computed_tokens_cpu
146
- context_lens_tensor = context_lens.to(query_start_loc.device)
146
+ context_lens_tensor = context_lens.to(query_start_loc.device, non_blocking=True)
147
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
148
149
if (