A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1][Minor] Set pin_memory=False for token_ids_cpu tensor (#11581)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 28, 2024 at 22:33 UTC
42bb201fd6f79d6ed2e28e0263ffa891cd993c4c
1 file changed
+3
-1
vllm/v1/worker/gpu_input_batch.py
+3
-1
@@ -57,11 +57,13 @@ class InputBatch:
57
58
# TODO(woosuk): This buffer could be too large if max_model_len is big.
59
# Find a way to reduce the CPU memory usage.
60
+ # This buffer is not directly transferred to the GPU, so it does not
61
+ # need to be pinned.
62
self.token_ids_cpu_tensor = torch.zeros(
63
(max_num_reqs, max_model_len),
64
device="cpu",
65
dtype=torch.int32,
64
- pin_memory=pin_memory,
66
+ pin_memory=False,
67
)
68
self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
69
self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)