A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 17:00 UTC
[Core][MM] Use non-blocking CPU-GPU copy of multimodal data (#28141)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Nov 6, 2025 at 04:05 UTC
80679f108ffd94c165ea11adbc3afcc43f24a06e
1 file changed
+3
-1
vllm/multimodal/utils.py
+3
-1
@@ -444,7 +444,9 @@ def group_mm_kwargs_by_modality(
444
445
if device is not None:
446
mm_kwargs_group = json_map_leaves(
447
- lambda x: x.to(device=device) if isinstance(x, torch.Tensor) else x,
447
+ lambda x: x.to(device=device, non_blocking=True)
448
+ if isinstance(x, torch.Tensor)
449
+ else x,
450
mm_kwargs_group,
451
)
452
else: