A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
fix: revert cast to cpu in `MsgpackEncoder._encode_tensor` to avoid hidden performance regressions (#25738)
Signed-off-by: Andrew Sansom <andrew@protopia.ai>
Andrew Sansom committed
Sep 26, 2025 at 03:18 UTC
e84e0735c71e983440b2194e0ea7f9117694dc71
2 files changed
+6
-1
vllm/inputs/preprocess.py
+5
@@ -278,6 +278,11 @@ class InputPreprocessor:
278
raise ValueError(
279
"prompt_embeds must be of shape (seq_len, hidden_size).")
280
281
+ # Tensors must be on CPU for serialization between processes
282
+ # in the MsgpackEncoder. Casting to CPU here ensures that there is no
283
+ # hidden device transfer in the critical path of generation.
284
+ prompt_embeds = prompt_embeds.cpu()
285
+
286
return embeds_inputs(prompt_embeds=prompt_embeds,
287
cache_salt=parsed_content.get("cache_salt"))
288
vllm/v1/serial_utils.py
+1
-1
@@ -208,7 +208,7 @@ class MsgpackEncoder:
208
) -> tuple[str, tuple[int, ...], Union[int, memoryview]]:
209
assert self.aux_buffers is not None
210
# view the tensor as a contiguous 1D array of bytes
211
- arr = obj.flatten().contiguous().cpu().view(torch.uint8).numpy()
211
+ arr = obj.flatten().contiguous().view(torch.uint8).numpy()
212
if obj.nbytes < self.size_threshold:
213
# Smaller tensors are encoded inline, just like ndarrays.
214
data = msgpack.Ext(CUSTOM_TYPE_RAW_VIEW, arr.data)