A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix] Update multimodel models mapping to fit new checkpoint after Transformers v4.52 (#19151)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jun 17, 2025 at 23:58 UTC
ca94d7fa007d113f2daf74e2f2278c03be676e93
12 files changed
+304
-75
tests/models/multimodal/test_mapping.py
new
+86
@@ -0,0 +1,86 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from collections.abc import Iterable
4
+
5
+import pytest
6
+import torch
7
+import transformers
8
+from transformers import AutoConfig, PreTrainedModel
9
+
10
+from vllm.config import ModelConfig
11
+from vllm.model_executor.models.utils import WeightsMapper
12
+from vllm.multimodal import MULTIMODAL_REGISTRY
13
+from vllm.transformers_utils.config import try_get_safetensors_metadata
14
+
15
+from ..registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS
16
+
17
+
18
+def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
19
+ """Create weights from safetensors checkpoint metadata"""
20
+ metadata = try_get_safetensors_metadata(repo)
21
+ weight_names = list(metadata.weight_map.keys())
22
+ with torch.device('meta'):
23
+ return ((name, torch.empty(0)) for name in weight_names)
24
+
25
+
26
+def create_model_dummy_weights(
27
+ repo: str,
28
+ model_arch: str,
29
+) -> Iterable[tuple[str, torch.Tensor]]:
30
+ """
31
+ Create weights from a dummy meta deserialized hf model with name conversion
32
+ """
33
+ model_cls: PreTrainedModel = getattr(transformers, model_arch)
34
+ config = AutoConfig.from_pretrained(repo)
35
+ with torch.device("meta"):
36
+ model: PreTrainedModel = model_cls._from_config(config)
37
+ return model.named_parameters()
38
+
39
+
40
+def model_architectures_for_test() -> list[str]:
41
+ arch_to_test = list[str]()
42
+ for model_arch, info in _MULTIMODAL_EXAMPLE_MODELS.items():
43
+ if not info.trust_remote_code and hasattr(transformers, model_arch):
44
+ model_cls: PreTrainedModel = getattr(transformers, model_arch)
45
+ if getattr(model_cls, "_checkpoint_conversion_mapping", None):
46
+ arch_to_test.append(model_arch)
47
+ return arch_to_test
48
+
49
+
50
+@pytest.mark.core_model
51
+@pytest.mark.parametrize("model_arch", model_architectures_for_test())
52
+def test_hf_model_weights_mapper(model_arch: str):
53
+ model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
54
+ model_info.check_available_online(on_fail="skip")
55
+ model_info.check_transformers_version(on_fail="skip")
56
+
57
+ model_id = model_info.default
58
+
59
+ model_config = ModelConfig(
60
+ model_id,
61
+ task="auto",
62
+ tokenizer=model_info.tokenizer or model_id,
63
+ tokenizer_mode=model_info.tokenizer_mode,
64
+ trust_remote_code=model_info.trust_remote_code,
65
+ seed=0,
66
+ dtype="auto",
67
+ revision=None,
68
+ hf_overrides=model_info.hf_overrides,
69
+ )
70
+ model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
71
+
72
+ original_weights = create_repo_dummy_weights(model_id)
73
+ hf_converted_weights = create_model_dummy_weights(model_id, model_arch)
74
+ mapper: WeightsMapper = model_cls.hf_to_vllm_mapper
75
+
76
+ mapped_original_weights = mapper.apply(original_weights)
77
+ mapped_hf_converted_weights = mapper.apply(hf_converted_weights)
78
+
79
+ ref_weight_names = set(map(lambda x: x[0], mapped_original_weights))
80
+ weight_names = set(map(lambda x: x[0], mapped_hf_converted_weights))
81
+
82
+ weights_missing = ref_weight_names - weight_names
83
+ weights_unmapped = weight_names - ref_weight_names
84
+ assert (not weights_missing and not weights_unmapped), (
85
+ f"Following weights are not mapped correctly: {weights_unmapped}, "
86
+ f"Missing expected weights: {weights_missing}.")
vllm/model_executor/models/aria.py
+5
@@ -486,6 +486,11 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
486
"""
487
hf_to_vllm_mapper = WeightsMapper(
488
orig_to_new_prefix={
489
+ # mapping for new names in checkpoint saved after transformers v4.52
490
+ "model.language_model.": "language_model.model.",
491
+ "model.vision_tower.": "vision_tower.",
492
+ "model.multi_modal_projector.": "multi_modal_projector.",
493
+ # mapping for original checkpoint
494
"language_model.model": "language_model",
495
"language_model.lm_head": "lm_head",
496
},
vllm/model_executor/models/aya_vision.py
+13
-3
@@ -32,8 +32,9 @@ from vllm.sequence import IntermediateTensors
32
33
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
34
from .siglip import SiglipVisionModel
35
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36
- maybe_prefix, merge_multimodal_embeddings)
35
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
36
+ init_vllm_registered_model, maybe_prefix,
37
+ merge_multimodal_embeddings)
38
39
40
class AyaVisionImagePixelInputs(TypedDict):
@@ -292,6 +293,15 @@ def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
293
class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
294
SupportsPP):
295
296
+ hf_to_vllm_mapper = WeightsMapper(
297
+ orig_to_new_prefix={
298
+ # mapping for new names in checkpoint saved after transformers v4.52
299
+ "model.language_model.": "language_model.model.",
300
+ "model.vision_tower.": "vision_tower.",
301
+ "model.multi_modal_projector.": "multi_modal_projector.",
302
+ "lm_head.": "language_model.lm_head.",
303
+ })
304
+
305
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
306
super().__init__()
307
config: AyaVisionConfig = vllm_config.model_config.hf_config
@@ -323,7 +333,7 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
333
def load_weights(self, weights: Iterable[tuple[str,
334
torch.Tensor]]) -> set[str]:
335
loader = AutoWeightsLoader(self)
326
- return loader.load_weights(weights)
336
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
337
338
def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
339
pixel_values: torch.Tensor,
vllm/model_executor/models/fuyu.py
+8
-1
@@ -42,7 +42,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
42
from vllm.sequence import IntermediateTensors
43
44
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
45
-from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
45
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,
46
merge_multimodal_embeddings)
47
48
# Cannot find the following 2 numbers from hf config.
@@ -245,6 +245,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
245
dummy_inputs=FuyuDummyInputsBuilder)
246
class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
247
248
+ hf_to_vllm_mapper = WeightsMapper(
249
+ orig_to_new_prefix={
250
+ "model.vision_embed_tokens.": "vision_embed_tokens.",
251
+ "model.language_model.": "language_model.model.",
252
+ "lm_head.": "language_model.lm_head.",
253
+ })
254
+
255
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
256
super().__init__()
257
config = vllm_config.model_config.hf_config
vllm/model_executor/models/gemma3_mm.py
+13
-3
@@ -35,8 +35,9 @@ from vllm.sequence import IntermediateTensors
35
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
36
SupportsMultiModal, SupportsPP)
37
from .siglip import SiglipVisionModel
38
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39
- maybe_prefix, merge_multimodal_embeddings)
38
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39
+ init_vllm_registered_model, maybe_prefix,
40
+ merge_multimodal_embeddings)
41
42
logger = init_logger(__name__)
43
@@ -471,6 +472,15 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
472
],
473
}
474
475
+ hf_to_vllm_mapper = WeightsMapper(
476
+ orig_to_new_prefix={
477
+ # mapping for new names in checkpoint saved after transformers v4.52
478
+ "model.language_model.": "language_model.model.",
479
+ "model.vision_tower.": "vision_tower.",
480
+ "model.multi_modal_projector.": "multi_modal_projector.",
481
+ "lm_head.": "language_model.lm_head.",
482
+ })
483
+
484
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
485
super().__init__()
486
config = vllm_config.model_config.hf_config
@@ -697,7 +707,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
707
def load_weights(self, weights: Iterable[tuple[str,
708
torch.Tensor]]) -> set[str]:
709
loader = AutoWeightsLoader(self)
700
- return loader.load_weights(weights)
710
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
711
712
def get_mm_mapping(self) -> MultiModelKeys:
713
"""
vllm/model_executor/models/llava.py
+13
-3
@@ -40,8 +40,9 @@ from .clip import CLIPVisionModel
40
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
41
from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
42
from .siglip import SiglipVisionModel
43
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
44
- maybe_prefix, merge_multimodal_embeddings)
43
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
44
+ init_vllm_registered_model, maybe_prefix,
45
+ merge_multimodal_embeddings)
46
from .vision import get_vision_encoder_info
47
48
@@ -499,6 +500,15 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
500
"gate_up_proj": ["gate_proj", "up_proj"]
501
}
502
503
+ hf_to_vllm_mapper = WeightsMapper(
504
+ orig_to_new_prefix={
505
+ # mapping for new names in checkpoint saved after transformers v4.52
506
+ "model.language_model.": "language_model.model.",
507
+ "model.vision_tower.": "vision_tower.",
508
+ "model.multi_modal_projector.": "multi_modal_projector.",
509
+ "lm_head.": "language_model.lm_head.",
510
+ })
511
+
512
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
513
super().__init__()
514
@@ -754,7 +764,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
764
def load_weights(self, weights: Iterable[tuple[str,
765
torch.Tensor]]) -> set[str]:
766
loader = AutoWeightsLoader(self)
757
- return loader.load_weights(weights)
767
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
768
769
770
class MantisProcessingInfo(LlavaProcessingInfo):
vllm/model_executor/models/llava_next.py
+13
-3
@@ -26,8 +26,8 @@ from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingInfo,
26
LlavaDummyInputsBuilder, LlavaLikeConfig,
27
LlavaMultiModalProjector, init_vision_tower_for_llava)
28
from .siglip import SiglipVisionModel
29
-from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
30
- init_vllm_registered_model, maybe_prefix)
29
+from .utils import (AutoWeightsLoader, WeightsMapper, embed_multimodal,
30
+ flatten_bn, init_vllm_registered_model, maybe_prefix)
31
32
33
class LlavaNextImagePixelInputs(TypedDict):
@@ -205,6 +205,16 @@ class LlavaNextMultiModalProcessor(
205
class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
206
SupportsPP):
207
208
+ hf_to_vllm_mapper = WeightsMapper(
209
+ orig_to_new_prefix={
210
+ # mapping for new names in checkpoint saved after transformers v4.52
211
+ "model.language_model.": "language_model.model.",
212
+ "model.vision_tower.": "vision_tower.",
213
+ "model.multi_modal_projector.": "multi_modal_projector.",
214
+ "model.image_newline": "image_newline",
215
+ "lm_head.": "language_model.lm_head.",
216
+ })
217
+
218
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
219
super().__init__()
220
config = vllm_config.model_config.hf_config
@@ -583,4 +593,4 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
593
def load_weights(self, weights: Iterable[tuple[str,
594
torch.Tensor]]) -> set[str]:
595
loader = AutoWeightsLoader(self)
586
- return loader.load_weights(weights)
596
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/llava_next_video.py
+14
-3
@@ -29,8 +29,9 @@ from vllm.utils import is_list_of
29
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
30
from .llava import init_vision_tower_for_llava
31
from .siglip import SiglipVisionModel
32
-from .utils import (AutoWeightsLoader, init_vllm_registered_model,
33
- maybe_prefix, merge_multimodal_embeddings)
32
+from .utils import (AutoWeightsLoader, WeightsMapper,
33
+ init_vllm_registered_model, maybe_prefix,
34
+ merge_multimodal_embeddings)
35
from .vision import get_vision_encoder_info
36
37
@@ -270,6 +271,16 @@ class LlavaNextMultiModalProjector(nn.Module):
271
class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
272
SupportsPP):
273
274
+ hf_to_vllm_mapper = WeightsMapper(
275
+ orig_to_new_prefix={
276
+ # mapping for new names in checkpoint saved after transformers v4.52
277
+ "model.language_model.": "language_model.model.",
278
+ "model.vision_tower.": "vision_tower.",
279
+ "model.multi_modal_projector.": "multi_modal_projector.",
280
+ "model.image_newline": "image_newline",
281
+ "lm_head.": "language_model.lm_head.",
282
+ })
283
+
284
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
285
super().__init__()
286
config = vllm_config.model_config.hf_config
@@ -468,4 +479,4 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
479
# This model doesn't support images for now
480
ignore_unexpected_prefixes=["image_newline"],
481
)
471
- return loader.load_weights(weights)
482
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/llava_onevision.py
+14
-3
@@ -30,8 +30,9 @@ from .llava import LlavaDummyInputsBuilder, init_vision_tower_for_llava
30
from .llava_next import (BaseLlavaNextMultiModalProcessor, LlavaNextLikeConfig,
31
LlavaNextProcessingInfo)
32
from .siglip import SiglipVisionModel
33
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
34
- maybe_prefix, merge_multimodal_embeddings)
33
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
34
+ init_vllm_registered_model, maybe_prefix,
35
+ merge_multimodal_embeddings)
36
37
# For profile run
38
_MAX_FRAMES_PER_VIDEO = 16
@@ -428,6 +429,16 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
429
class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
430
SupportsPP):
431
432
+ hf_to_vllm_mapper = WeightsMapper(
433
+ orig_to_new_prefix={
434
+ # mapping for new names in checkpoint saved after transformers v4.52
435
+ "model.language_model.": "language_model.model.",
436
+ "model.vision_tower.": "vision_tower.",
437
+ "model.multi_modal_projector.": "multi_modal_projector.",
438
+ "model.image_newline": "image_newline",
439
+ "lm_head.": "language_model.lm_head.",
440
+ })
441
+
442
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
443
super().__init__()
444
config = vllm_config.model_config.hf_config
@@ -954,4 +965,4 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
965
def load_weights(self, weights: Iterable[tuple[str,
966
torch.Tensor]]) -> set[str]:
967
loader = AutoWeightsLoader(self)
957
- return loader.load_weights(weights)
968
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/mistral3.py
+13
-3
@@ -36,8 +36,9 @@ from vllm.sequence import IntermediateTensors
36
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
37
SupportsMultiModal, SupportsPP)
38
from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
39
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
40
- maybe_prefix, merge_multimodal_embeddings)
39
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
40
+ init_vllm_registered_model, maybe_prefix,
41
+ merge_multimodal_embeddings)
42
from .vision import get_vision_encoder_info
43
44
@@ -389,6 +390,15 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,
390
"gate_up_proj": ["gate_proj", "up_proj"]
391
}
392
393
+ hf_to_vllm_mapper = WeightsMapper(
394
+ orig_to_new_prefix={
395
+ # mapping for new names in checkpoint saved after transformers v4.52
396
+ "model.language_model.": "language_model.model.",
397
+ "model.vision_tower.": "vision_tower.",
398
+ "model.multi_modal_projector.": "multi_modal_projector.",
399
+ "lm_head.": "language_model.lm_head.",
400
+ })
401
+
402
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
403
super().__init__()
404
@@ -592,7 +602,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,
602
def load_weights(self, weights: Iterable[tuple[str,
603
torch.Tensor]]) -> set[str]:
604
loader = AutoWeightsLoader(self)
595
- return loader.load_weights(weights)
605
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
606
607
def get_mm_mapping(self) -> MultiModelKeys:
608
"""
vllm/model_executor/models/mllama.py
+99
-50
@@ -67,7 +67,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
67
from .clip import CLIPMLP
68
from .interfaces import SupportsMultiModal, SupportsV0Only
69
from .llama import LlamaDecoderLayer, LlamaMLP
70
-from .utils import maybe_prefix
70
+from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
71
72
logger = init_logger(__name__)
73
@@ -790,6 +790,36 @@ class MllamaVisionModel(nn.Module):
790
dim=-1)
791
return hidden_state
792
793
+ def load_weights(self, weights: Iterable[tuple[str,
794
+ torch.Tensor]]) -> set[str]:
795
+ stacked_params_mapping = [
796
+ # (param_name, shard_name, shard_id)
797
+ (".qkv_proj", ".q_proj", "q"),
798
+ (".qkv_proj", ".k_proj", "k"),
799
+ (".qkv_proj", ".v_proj", "v"),
800
+ ]
801
+ params_dict = dict(self.named_parameters())
802
+ updated_params: set[str] = set()
803
+ for name, loaded_weight in weights:
804
+ if 'patch_embedding._linear.weight' in name:
805
+ loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
806
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
807
+ if weight_name not in name:
808
+ continue
809
+ name = name.replace(weight_name, param_name)
810
+ param = params_dict[name]
811
+ updated_params.add(name)
812
+ weight_loader = param.weight_loader
813
+ weight_loader(param, loaded_weight, shard_id)
814
+ break
815
+ else:
816
+ param = params_dict.pop(name)
817
+ weight_loader = getattr(param, "weight_loader",
818
+ default_weight_loader)
819
+ weight_loader(param, loaded_weight)
820
+ updated_params.add(name)
821
+ return updated_params
822
+
823
824
class MllamaTextRMSNorm(nn.Module):
825
@@ -1132,6 +1162,7 @@ class MllamaForCausalLM(nn.Module):
1162
1163
config = vllm_config.model_config.hf_config.text_config
1164
quant_config = vllm_config.quant_config
1165
+ self.quant_config = quant_config
1166
1167
self.vocab_size = config.vocab_size
1168
self.model = MllamaTextModel(vllm_config=vllm_config,
@@ -1167,6 +1198,58 @@ class MllamaForCausalLM(nn.Module):
1198
)
1199
return hidden_states
1200
1201
+ def load_weights(self, weights: Iterable[tuple[str,
1202
+ torch.Tensor]]) -> set[str]:
1203
+ stacked_params_mapping = [
1204
+ # (param_name, shard_name, shard_id)
1205
+ (".qkv_proj", ".q_proj", "q"),
1206
+ (".qkv_proj", ".k_proj", "k"),
1207
+ (".qkv_proj", ".v_proj", "v"),
1208
+ (".gate_up_proj", ".gate_proj", 0),
1209
+ (".gate_up_proj", ".up_proj", 1),
1210
+ ]
1211
+ params_dict = dict(self.named_parameters())
1212
+ updated_params: set[str] = set()
1213
+ for name, loaded_weight in weights:
1214
+ if 'patch_embedding.weight' in name:
1215
+ name = name.replace('patch_embedding.weight',
1216
+ 'patch_embedding._linear.weight')
1217
+ loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1218
+ if (self.quant_config is not None and
1219
+ (scale_name := self.quant_config.get_cache_scale(name))):
1220
+ # Loading kv cache quantization scales
1221
+ param = params_dict[scale_name]
1222
+ weight_loader = getattr(param, "weight_loader",
1223
+ default_weight_loader)
1224
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
1225
+ loaded_weight[0])
1226
+ weight_loader(param, loaded_weight)
1227
+ updated_params.add(scale_name)
1228
+ continue
1229
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
1230
+ if weight_name not in name:
1231
+ continue
1232
+ name = name.replace(weight_name, param_name)
1233
+ param = params_dict[name]
1234
+ updated_params.add(name)
1235
+ weight_loader = param.weight_loader
1236
+ weight_loader(param, loaded_weight, shard_id)
1237
+ break
1238
+ else:
1239
+ orig_name = name
1240
+ name = maybe_remap_kv_scale_name(name, params_dict)
1241
+ if name is None:
1242
+ logger.debug("Missing name %s, orig name %s", name,
1243
+ orig_name)
1244
+ continue
1245
+
1246
+ param = params_dict.pop(name)
1247
+ weight_loader = getattr(param, "weight_loader",
1248
+ default_weight_loader)
1249
+ weight_loader(param, loaded_weight)
1250
+ updated_params.add(name)
1251
+ return updated_params
1252
+
1253
1254
@MULTIMODAL_REGISTRY.register_processor(MllamaMultiModalProcessor,
1255
info=MllamaProcessingInfo,
@@ -1178,6 +1261,19 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,
1261
"gate_up_proj": ["gate_proj", "up_proj"]
1262
}
1263
1264
+ hf_to_vllm_mapper = WeightsMapper(
1265
+ orig_to_new_prefix={
1266
+ # mapping for new names in checkpoint saved after transformers v4.52
1267
+ "model.vision_model.": "vision_model.",
1268
+ "model.multi_modal_projector.": "multi_modal_projector.",
1269
+ "model.language_model.": "language_model.model.",
1270
+ "lm_head.": "language_model.lm_head.",
1271
+ },
1272
+ orig_to_new_suffix={
1273
+ "patch_embedding.weight": "patch_embedding._linear.weight",
1274
+ },
1275
+ )
1276
+
1277
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1278
super().__init__()
1279
config: MllamaConfig = vllm_config.model_config.hf_config
@@ -1479,55 +1575,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,
1575
1576
def load_weights(self, weights: Iterable[tuple[str,
1577
torch.Tensor]]) -> set[str]:
1482
- stacked_params_mapping = [
1483
- # (param_name, shard_name, shard_id)
1484
- (".qkv_proj", ".q_proj", "q"),
1485
- (".qkv_proj", ".k_proj", "k"),
1486
- (".qkv_proj", ".v_proj", "v"),
1487
- (".gate_up_proj", ".gate_proj", 0),
1488
- (".gate_up_proj", ".up_proj", 1),
1489
- ]
1490
- params_dict = dict(self.named_parameters())
1491
- updated_params: set[str] = set()
1492
- for name, loaded_weight in weights:
1493
- if 'patch_embedding.weight' in name:
1494
- name = name.replace('patch_embedding.weight',
1495
- 'patch_embedding._linear.weight')
1496
- loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1497
- if (self.quant_config is not None and
1498
- (scale_name := self.quant_config.get_cache_scale(name))):
1499
- # Loading kv cache quantization scales
1500
- param = params_dict[scale_name]
1501
- weight_loader = getattr(param, "weight_loader",
1502
- default_weight_loader)
1503
- loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
1504
- loaded_weight[0])
1505
- weight_loader(param, loaded_weight)
1506
- updated_params.add(scale_name)
1507
- continue
1508
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
1509
- if weight_name not in name:
1510
- continue
1511
- name = name.replace(weight_name, param_name)
1512
- param = params_dict[name]
1513
- updated_params.add(name)
1514
- weight_loader = param.weight_loader
1515
- weight_loader(param, loaded_weight, shard_id)
1516
- break
1517
- else:
1518
- orig_name = name
1519
- name = maybe_remap_kv_scale_name(name, params_dict)
1520
- if name is None:
1521
- logger.debug("Missing name %s, orig name %s", name,
1522
- orig_name)
1523
- continue
1524
-
1525
- param = params_dict.pop(name)
1526
- weight_loader = getattr(param, "weight_loader",
1527
- default_weight_loader)
1528
- weight_loader(param, loaded_weight)
1529
- updated_params.add(name)
1530
- return updated_params
1578
+ loader = AutoWeightsLoader(self)
1579
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1580
1581
def get_mm_mapping(self) -> MultiModelKeys:
1582
"""
vllm/model_executor/models/paligemma.py
+13
-3
@@ -24,8 +24,9 @@ from vllm.sequence import IntermediateTensors
24
25
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
26
from .siglip import SiglipVisionModel
27
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
28
- maybe_prefix, merge_multimodal_embeddings)
27
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
28
+ init_vllm_registered_model, maybe_prefix,
29
+ merge_multimodal_embeddings)
30
from .vision import get_vision_encoder_info
31
32
logger = init_logger(__name__)
@@ -227,6 +228,15 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
228
],
229
}
230
231
+ hf_to_vllm_mapper = WeightsMapper(
232
+ orig_to_new_prefix={
233
+ # mapping for new names in checkpoint saved after transformers v4.52
234
+ "model.language_model.": "language_model.model.",
235
+ "model.vision_tower.": "vision_tower.",
236
+ "model.multi_modal_projector.": "multi_modal_projector.",
237
+ "lm_head.": "language_model.lm_head.",
238
+ })
239
+
240
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
241
super().__init__()
242
config = vllm_config.model_config.hf_config
@@ -395,4 +405,4 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
405
def load_weights(self, weights: Iterable[tuple[str,
406
torch.Tensor]]) -> set[str]:
407
loader = AutoWeightsLoader(self)
398
- return loader.load_weights(weights)
408
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)