A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 13:30 UTC
[Bug] Fix `tests/distributed/test_elastic_ep.py - assert False` (#43813)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
May 28, 2026 at 11:00 UTC
be4062fd6c56ba5528a8e0d3b0bbe1e2be8d22e9
1 file changed
+6
-2
vllm/v1/utils.py
+6
-2
@@ -117,8 +117,12 @@ class CpuGpuBuffer:
117
pin_memory: bool,
118
with_numpy: bool = True,
119
) -> None:
120
- self.cpu = torch.zeros(*size, dtype=dtype, device="cpu", pin_memory=pin_memory)
121
- self.gpu = torch.zeros_like(self.cpu, device=device)
120
+ # these buffers are mutable runtime state, so allocate them as normal
121
+ with torch.inference_mode(False):
122
+ self.cpu = torch.zeros(
123
+ *size, dtype=dtype, device="cpu", pin_memory=pin_memory
124
+ )
125
+ self.gpu = torch.zeros_like(self.cpu, device=device)
126
self.np: np.ndarray
127
# To keep type hints simple (avoiding generics and subclasses), we
128
# only conditionally create the numpy array attribute. This can cause