A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 13:45 UTC
fix(gguf): Disable bfloat16 for GGUF on blackwell device (#30408)
Signed-off-by: Christina <truffle@gmail.com> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: Christina Norman <christina@example.com> Co-authored-by: Isotr0py <isotr0py@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Christina Norman committed
Dec 12, 2025 at 09:10 UTC
dc13c99eedf837f22f60e9b5836abf147f5254f1
1 file changed
+6
vllm/model_executor/layers/quantization/gguf.py
+6
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
33
)
34
from vllm.model_executor.models.utils import WeightsMapper
35
from vllm.model_executor.utils import set_weight_attrs
36
+from vllm.platforms import current_platform
37
from vllm.utils.torch_utils import direct_register_custom_op
38
39
logger = init_logger(__name__)
@@ -52,6 +53,11 @@ class GGUFConfig(QuantizationConfig):
53
return "gguf"
54
55
def get_supported_act_dtypes(self) -> list[torch.dtype]:
56
+ # GGUF dequantization kernels use half precision (fp16) internally.
57
+ # bfloat16 has precision issues on Blackwell devices.
58
+ if current_platform.has_device_capability(100):
59
+ logger.warning_once("GGUF has precision issues with bfloat16 on Blackwell.")
60
+ return [torch.half, torch.float32]
61
return [torch.half, torch.bfloat16, torch.float32]
62
63
@classmethod