A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Asynchronous h2d in merge_multimodal_embeddings via pinned memory. (#23686)
Signed-off-by: Chenheli Hua <huachenheli@outlook.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Chenheli Hua committed
Aug 26, 2025 at 20:05 UTC
c905684cfeaee3b2be2c736eee473b2c6ae7f7bf
1 file changed
+3
-1
vllm/model_executor/models/utils.py
+3
-1
@@ -508,7 +508,9 @@ def merge_multimodal_embeddings(
508
"""
509
if isinstance(placeholder_token_id, list):
510
placeholder_token_id = torch.tensor(placeholder_token_id,
511
- device=input_ids.device)
511
+ pin_memory=True).to(
512
+ device=input_ids.device,
513
+ non_blocking=True)
514
return _merge_multimodal_embeddings(
515
inputs_embeds,
516
torch.isin(input_ids, placeholder_token_id),