@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 42bb201fd6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1][Minor] Set pin_memory=False for token_ids_cpu tensor (#11581)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 28, 2024 at 22:33 UTC 42bb201fd6f79d6ed2e28e0263ffa891cd993c4c
1 file changed +3 -1
vllm/v1/worker/gpu_input_batch.py
+3 -1
@@ -57,11 +57,13 @@ class InputBatch:
57
58 # TODO(woosuk): This buffer could be too large if max_model_len is big.
59 # Find a way to reduce the CPU memory usage.
60 + # This buffer is not directly transferred to the GPU, so it does not
61 + # need to be pinned.
62 self.token_ids_cpu_tensor = torch.zeros(
63 (max_num_reqs, max_model_len),
64 device="cpu",
65 dtype=torch.int32,
64 - pin_memory=pin_memory,
66 + pin_memory=False,
67 )
68 self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
69 self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)