A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI][Bugfix] Fix failing Blackwell test (#24993)
Signed-off-by: Matthew Bonanni <mbonanni001@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Matthew Bonanni committed
Sep 16, 2025 at 18:55 UTC
d119fc86140785e7efc8f125c17153544d1e0f20
1 file changed
+2
-5
vllm/model_executor/layers/fused_moe/modular_kernel.py
+2
-5
@@ -506,12 +506,9 @@ class SharedResizableBuffer:
506
def get(self, shape: tuple[int, ...], device: torch.device,
507
dtype: torch.dtype):
508
shape_numel = prod(shape)
509
- if self.buffer is None or self.buffer.numel() < shape_numel:
509
+ if (self.buffer is None or self.buffer.numel() < shape_numel
510
+ or self.buffer.device != device or self.buffer.dtype != dtype):
511
self.buffer = torch.empty(shape_numel, device=device, dtype=dtype)
511
- assert self.buffer.device == device, \
512
- f"Buffer device mismatch: {self.buffer.device} != {device}"
513
- assert self.buffer.dtype == dtype, \
514
- f"Buffer dtype mismatch: {self.buffer.dtype} != {dtype}"
512
return self.buffer[:shape_numel].view(*shape)
513
514