A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 20:00 UTC
[Bugfix] Fix quantized model initialization failure with prefetch offloading (#40432)
Signed-off-by: Rishapveer Singh <singhrishapveer@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Rishapveer Singh committed
Apr 22, 2026 at 05:15 UTC
9b60e2ffaae1571f07dd4739da943415be8f4da5
1 file changed
+2
-1
vllm/model_executor/offloader/prefetch.py
+2
-1
@@ -21,6 +21,7 @@ import torch.nn as nn
21
import vllm.model_executor.offloader.prefetch_ops # noqa: F401
22
from vllm.logger import init_logger
23
from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory
24
+from vllm.utils.torch_utils import get_dtype_size
25
26
logger = init_logger(__name__)
27
@@ -53,7 +54,7 @@ class ParamInfo:
54
numel = 1
55
for dim in self.shape:
56
numel *= dim
56
- return numel * torch.finfo(self.dtype).bits // 8
57
+ return numel * get_dtype_size(self.dtype)
58
59
60
class StaticBufferPool: