A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 03:30 UTC
[Bugfix][Quantization] Support BF16 tensors on GGUF (#29948)
Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com>
Tsukasa OI committed
Dec 3, 2025 at 19:33 UTC
42c194964341bea9fc59e0d35db04dfafc3c473d
2 files changed
+18
-1
tests/models/quantization/test_gguf.py
+7
@@ -47,6 +47,12 @@ QWEN2_CONFIG = GGUFTestConfig(
47
gguf_filename="qwen2.5-1.5b-instruct-q6_k.gguf",
48
)
49
50
+QWEN3_CONFIG = GGUFTestConfig(
51
+ original_model="Qwen/Qwen3-0.6B",
52
+ gguf_repo="unsloth/Qwen3-0.6B-GGUF",
53
+ gguf_filename="Qwen3-0.6B-BF16.gguf",
54
+)
55
+
56
PHI3_CONFIG = GGUFTestConfig(
57
original_model="microsoft/Phi-3.5-mini-instruct",
58
gguf_repo="bartowski/Phi-3.5-mini-instruct-GGUF",
@@ -87,6 +93,7 @@ GEMMA3_CONFIG = GGUFTestConfig(
93
MODELS = [
94
# LLAMA_CONFIG, # broken: https://github.com/vllm-project/vllm/issues/19458
95
QWEN2_CONFIG,
96
+ QWEN3_CONFIG,
97
PHI3_CONFIG,
98
GPT2_CONFIG,
99
STABLELM_CONFIG,
vllm/model_executor/model_loader/weight_utils.py
+11
-1
@@ -921,7 +921,17 @@ def gguf_quant_weights_iterator(
921
name = gguf_to_hf_name_map[tensor.name]
922
if weight_type.name not in ("F32", "BF16", "F16"):
923
name = name.replace("weight", "qweight")
924
- param = torch.tensor(weight)
924
+ if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:
925
+ # BF16 is currently the only "quantization" type that isn't
926
+ # actually quantized but is read as a raw byte tensor.
927
+ # Reinterpret as `torch.bfloat16` tensor.
928
+ weight = weight.view(np.uint16)
929
+ if reader.byte_order == "S":
930
+ # GGUF endianness != system endianness
931
+ weight = weight.byteswap()
932
+ param = torch.tensor(weight).view(torch.bfloat16)
933
+ else:
934
+ param = torch.tensor(weight)
935
yield name, param
936
937