A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] CohereModel.load_weights: skip modelopt _quantizer.* keys (#43495)
Signed-off-by: Kai Köhler <kai.koehler@web.de>
Kai K. committed
Jun 11, 2026 at 17:14 UTC
f1d8d99717b6aebf19eac459e0c1fd04bdbe356c
1 file changed
+5
-1
vllm/model_executor/models/commandr.py
+5
-1
@@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors
56
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
57
from .utils import (
58
AutoWeightsLoader,
59
+ WeightsMapper,
60
extract_layer_index,
61
is_pp_missing_parameter,
62
make_empty_intermediate_tensors_factory,
@@ -397,6 +398,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
398
}
399
# LoRA specific attributes
400
embedding_modules = {"embed_tokens": "input_embeddings"}
401
+ # ModelOpt NVFP4 checkpoints carry raw quantizer-module state
402
+ # (e.g. "*.weight_quantizer._double_scale"); drop them before loading. See #41925.
403
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={"_quantizer.": None})
404
405
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
406
super().__init__()
@@ -453,4 +457,4 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
457
loader = AutoWeightsLoader(
458
self, skip_prefixes=["lm_head", "rotary_emb.inv_freq"]
459
)
456
- return loader.load_weights(weights)
460
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)