A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Gemma3: Fix GGUF loading and quantization (#26189)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Luciano Martins committed
Oct 9, 2025 at 04:00 UTC
1317028aa8bc9710532987b27dd044bcea2c161f
2 files changed
+14
vllm/model_executor/model_loader/gguf_loader.py
+4
@@ -72,6 +72,10 @@ class GGUFModelLoader(BaseModelLoader):
72
# hack: ggufs have a different name than transformers
73
if model_type == "cohere":
74
model_type = "command-r"
75
+ if model_type == "gemma3_text":
76
+ # Gemma3 models use "gemma3_text" in HuggingFace but
77
+ # "gemma3" in GGUF architecture naming
78
+ model_type = "gemma3"
79
if model_type in ("deepseek_v3", "deepseek_v2"):
80
model_type = "deepseek2"
81
# GGUF layer map assumes that we will have a merged expert weights
vllm/model_executor/models/gemma3.py
+10
@@ -372,6 +372,7 @@ class Gemma3Model(nn.Module):
372
self.embed_tokens = VocabParallelEmbedding(
373
config.vocab_size,
374
config.hidden_size,
375
+ quant_config=quant_config,
376
prefix=f"{prefix}.embed_tokens",
377
)
378
self.start_layer, self.end_layer, self.layers = make_layers(
@@ -442,6 +443,15 @@ class Gemma3Model(nn.Module):
443
params_dict = dict(self.named_parameters())
444
loaded_params: set[str] = set()
445
for name, loaded_weight in weights:
446
+ # Revert +1 during llama.cpp conversion
447
+ # see: https://github.com/ggml-org/llama.cpp/blob/be7c3034108473beda214fd1d7c98fd6a7a3bdf5/convert_hf_to_gguf.py#L3397-L3400
448
+ if (
449
+ self.quant_config
450
+ and self.quant_config.get_name() == "gguf"
451
+ and name.endswith("norm.weight")
452
+ ):
453
+ loaded_weight -= 1
454
+
455
if self.quant_config is not None and (
456
scale_name := self.quant_config.get_cache_scale(name)
457
):