A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 18:15 UTC
[V1] Remove pin_memory() in async_copy_to_gpu to fix sporadic stalls (#37006)
Signed-off-by: Sebastien Beurnier <sbeurnier@together.ai>
sbeurnier committed
Mar 14, 2026 at 02:37 UTC
a116f969301acfdb6ea9fa917815566d434fdc95
1 file changed
+4
-6
vllm/v1/worker/gpu/buffer_utils.py
+4
-6
@@ -27,12 +27,10 @@ def async_copy_to_gpu(
27
assert device is not None
28
out = torch.empty_like(x, device=device)
29
30
- # CPU-to-CPU copy
31
- tmp = x.pin_memory()
32
- assert tmp is not x
33
-
34
- # CPU-to-GPU copy
35
- return out.copy_(tmp, non_blocking=True)
30
+ # Copy directly to GPU — explicit pin_memory() causes sporadic stalls
31
+ # under high concurrency due to CUDA driver contention. The driver
32
+ # handles the transfer efficiently without manual pinning.
33
+ return out.copy_(x, non_blocking=True)
34
35
36
class UvaBuffer: