Forward fix nightly errors from #44589 (#47151)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jun 30, 2026 at 15:02 UTC
62c7d8009f33e3cd45b09bbd57dc0a2adca1a949
3 files changed
+27
-17
vllm/model_executor/models/commandr.py
+5
-4
@@ -348,7 +348,11 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
348
".v_proj": (".qkv_proj", "v"),
349
".gate_proj": (".gate_up_proj", 0),
350
".up_proj": (".gate_up_proj", 1),
351
- }
351
+ },
352
+ # ModelOpt NVFP4 checkpoints carry raw quantizer-module state
353
+ # (e.g. "*.weight_quantizer._double_scale"); drop them before loading.
354
+ # See #41925.
355
+ orig_to_new_substr={"_quantizer.": None},
356
)
357
packed_modules_mapping = {
358
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -356,9 +360,6 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
360
}
361
# LoRA specific attributes
362
embedding_modules = {"embed_tokens": "input_embeddings"}
359
- # ModelOpt NVFP4 checkpoints carry raw quantizer-module state
360
- # (e.g. "*.weight_quantizer._double_scale"); drop them before loading. See #41925.
361
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={"_quantizer.": None})
363
364
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
365
super().__init__()
vllm/model_executor/models/gemma3.py
+16
-10
@@ -291,6 +291,17 @@ class Gemma3DecoderLayer(nn.Module):
291
292
@support_torch_compile
293
class Gemma3Model(nn.Module):
294
+ hf_to_vllm_mapper = WeightsMapper(
295
+ orig_to_new_stacked={
296
+ # weight_name: (param_name, shard_id)
297
+ ".q_proj": (".qkv_proj", "q"),
298
+ ".k_proj": (".qkv_proj", "k"),
299
+ ".v_proj": (".qkv_proj", "v"),
300
+ ".gate_proj": (".gate_up_proj", 0),
301
+ ".up_proj": (".gate_up_proj", 1),
302
+ }
303
+ )
304
+
305
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
306
super().__init__()
307
config = vllm_config.model_config.hf_config
@@ -361,18 +372,13 @@ class Gemma3Model(nn.Module):
372
hidden_states, _ = self.norm(hidden_states, residual)
373
return hidden_states
374
375
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
376
+ loader = AutoWeightsLoader(self)
377
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
378
+
379
380
class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
366
- hf_to_vllm_mapper = WeightsMapper(
367
- orig_to_new_stacked={
368
- # weight_name: (param_name, shard_id)
369
- ".q_proj": (".qkv_proj", "q"),
370
- ".k_proj": (".qkv_proj", "k"),
371
- ".v_proj": (".qkv_proj", "v"),
372
- ".gate_proj": (".gate_up_proj", 0),
373
- ".up_proj": (".gate_up_proj", 1),
374
- }
375
- )
381
+ hf_to_vllm_mapper = Gemma3Model.hf_to_vllm_mapper
382
packed_modules_mapping = {
383
"qkv_proj": [
384
"q_proj",
vllm/model_executor/models/jina.py
+6
-3
@@ -25,7 +25,7 @@ from ..layers.pooler.tokwise import (
25
from .interfaces import SupportsLateInteraction
26
from .interfaces_base import VllmModelForPooling
27
from .qwen3 import Qwen3ForCausalLM, Qwen3Model
28
-from .utils import AutoWeightsLoader, maybe_prefix
28
+from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
29
30
logger = logging.getLogger(__name__)
31
@@ -193,6 +193,9 @@ class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling):
193
"""
194
195
is_pooling_model = True
196
+ hf_to_vllm_mapper = Qwen3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
197
+ orig_to_new_prefix={"": "model."}
198
+ )
199
200
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
201
super().__init__(vllm_config=vllm_config, prefix=prefix)
@@ -254,6 +257,6 @@ class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling):
257
tensor = tensor + (lora_B @ lora_A) * scaling
258
yield name, tensor
259
257
- loader = AutoWeightsLoader(self.model, ignore_unexpected_prefixes=["lm_head."])
260
+ loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["lm_head."])
261
weights = _merge_weights(weights)
259
- return loader.load_weights(weights, mapper=self.model.hf_to_vllm_mapper)
262
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)