| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from collections.abc import Iterable |
| 4 | |
| 5 | import pytest |
| 6 | import torch |
| 7 | import transformers |
| 8 | from transformers import AutoConfig, AutoModel, PreTrainedModel |
| 9 | |
| 10 | from vllm.config import ModelConfig |
| 11 | from vllm.model_executor.models.transformers.base import Base as TransformersBase |
| 12 | from vllm.model_executor.models.utils import WeightsMapper |
| 13 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 14 | from vllm.transformers_utils.config import try_get_safetensors_metadata |
| 15 | |
| 16 | from ..registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS |
| 17 | |
| 18 | |
| 19 | def test_cosmos3_new_checkpoint_weights_mapper(): |
| 20 | from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration |
| 21 | |
| 22 | mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper |
| 23 | |
| 24 | assert mapper.apply_list( |
| 25 | [ |
| 26 | "layers.0.self_attn.to_q.weight", |
| 27 | "layers.0.self_attn.to_k.weight", |
| 28 | "layers.0.self_attn.to_v.weight", |
| 29 | "layers.0.self_attn.to_out.weight", |
| 30 | "layers.0.self_attn.norm_q.weight", |
| 31 | "layers.0.self_attn.norm_k.weight", |
| 32 | "embed_tokens.weight", |
| 33 | "norm.weight", |
| 34 | "lm_head.weight", |
| 35 | "blocks.0.attn.qkv.weight", |
| 36 | ] |
| 37 | ) == [ |
| 38 | "language_model.model.layers.0.self_attn.q_proj.weight", |
| 39 | "language_model.model.layers.0.self_attn.k_proj.weight", |
| 40 | "language_model.model.layers.0.self_attn.v_proj.weight", |
| 41 | "language_model.model.layers.0.self_attn.o_proj.weight", |
| 42 | "language_model.model.layers.0.self_attn.q_norm.weight", |
| 43 | "language_model.model.layers.0.self_attn.k_norm.weight", |
| 44 | "language_model.model.embed_tokens.weight", |
| 45 | "language_model.model.norm.weight", |
| 46 | "language_model.lm_head.weight", |
| 47 | "visual.blocks.0.attn.qkv.weight", |
| 48 | ] |
| 49 | |
| 50 | assert ( |
| 51 | mapper.apply_list( |
| 52 | [ |
| 53 | "layers.0.self_attn.add_q_proj.weight", |
| 54 | "layers.0.self_attn.add_k_proj.weight", |
| 55 | "layers.0.self_attn.add_v_proj.weight", |
| 56 | "layers.0.self_attn.to_add_out.weight", |
| 57 | "layers.0.self_attn.norm_added_q.weight", |
| 58 | "layers.0.self_attn.norm_added_k.weight", |
| 59 | "layers.0.self_attn.q_proj_moe_gen.weight", |
| 60 | "layers.0.mlp_moe_gen.gate_up_proj.weight", |
| 61 | "norm_moe_gen.weight", |
| 62 | "proj_in.weight", |
| 63 | "proj_out.weight", |
| 64 | "time_embedder.linear_1.weight", |
| 65 | "audio_proj_in.weight", |
| 66 | "audio_proj_out.weight", |
| 67 | "action_proj_in.weight", |
| 68 | "action_proj_out.weight", |
| 69 | "audio_modality_embed", |
| 70 | "action_modality_embed", |
| 71 | ] |
| 72 | ) |
| 73 | == [] |
| 74 | ) |
| 75 | |
| 76 | |
| 77 | def test_cosmos3_modelopt_quantizer_weights_mapper(): |
| 78 | """ModelOpt/Diffusers FP8 checkpoints ship native fake-quant buffers |
| 79 | (``*_quantizer._amax`` / ``._scale``) alongside the vLLM-consumable |
| 80 | ``weight_scale`` / ``input_scale`` sidecars. vLLM must drop the former |
| 81 | (it has no parameter for them) while keeping the latter.""" |
| 82 | from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration |
| 83 | |
| 84 | mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper |
| 85 | |
| 86 | # Native ModelOpt quantizer buffers are dropped. |
| 87 | assert ( |
| 88 | mapper.apply_list( |
| 89 | [ |
| 90 | "layers.0.self_attn.to_q.input_quantizer._amax", |
| 91 | "layers.0.self_attn.to_q.weight_quantizer._amax", |
| 92 | "layers.0.self_attn.to_q.weight_quantizer._scale", |
| 93 | "layers.0.mlp.down_proj.output_quantizer._amax", |
| 94 | ] |
| 95 | ) |
| 96 | == [] |
| 97 | ) |
| 98 | |
| 99 | # The FP8 scale sidecars vLLM actually consumes are kept and remapped. |
| 100 | assert mapper.apply_list( |
| 101 | [ |
| 102 | "layers.0.self_attn.to_q.weight", |
| 103 | "layers.0.self_attn.to_q.weight_scale", |
| 104 | "layers.0.self_attn.to_q.input_scale", |
| 105 | "layers.0.mlp.down_proj.input_scale", |
| 106 | ] |
| 107 | ) == [ |
| 108 | "language_model.model.layers.0.self_attn.q_proj.weight", |
| 109 | "language_model.model.layers.0.self_attn.q_proj.weight_scale", |
| 110 | "language_model.model.layers.0.self_attn.q_proj.input_scale", |
| 111 | "language_model.model.layers.0.mlp.down_proj.input_scale", |
| 112 | ] |
| 113 | |
| 114 | |
| 115 | def test_cosmos3_edge_checkpoint_weights_mapper(): |
| 116 | from vllm.model_executor.models.cosmos3_edge import ( |
| 117 | Cosmos3EdgeForConditionalGeneration, |
| 118 | ) |
| 119 | |
| 120 | mapper = Cosmos3EdgeForConditionalGeneration.hf_to_vllm_mapper |
| 121 | |
| 122 | assert mapper.apply_list( |
| 123 | [ |
| 124 | "embed_tokens.weight", |
| 125 | "norm.weight", |
| 126 | "layers.0.input_layernorm.weight", |
| 127 | "layers.0.self_attn.to_q.weight", |
| 128 | "layers.0.self_attn.to_k.weight", |
| 129 | "layers.0.self_attn.to_v.weight", |
| 130 | "layers.0.self_attn.to_out.weight", |
| 131 | "layers.0.post_attention_layernorm.weight", |
| 132 | "layers.0.mlp.up_proj.weight", |
| 133 | "layers.0.mlp.down_proj.weight", |
| 134 | "layers.27.input_layernorm.weight", |
| 135 | "layers.27.self_attn.to_q.weight", |
| 136 | "layers.27.post_attention_layernorm.weight", |
| 137 | "layers.27.mlp.down_proj.weight", |
| 138 | "model.visual.embeddings.patch_embedding.weight", |
| 139 | "model.visual.encoder.layers.0.self_attn.q_proj.weight", |
| 140 | "model.projector.linear_fc1.weight", |
| 141 | "lm_head.weight", |
| 142 | ] |
| 143 | ) == [ |
| 144 | "language_model.model.embed_tokens.weight", |
| 145 | "language_model.model.norm_f.weight", |
| 146 | "language_model.model.layers.0.norm.weight", |
| 147 | "language_model.model.layers.0.mixer.qkv_proj.weight", |
| 148 | "language_model.model.layers.0.mixer.qkv_proj.weight", |
| 149 | "language_model.model.layers.0.mixer.qkv_proj.weight", |
| 150 | "language_model.model.layers.0.mixer.o_proj.weight", |
| 151 | "language_model.model.layers.1.norm.weight", |
| 152 | "language_model.model.layers.1.mixer.up_proj.weight", |
| 153 | "language_model.model.layers.1.mixer.down_proj.weight", |
| 154 | "language_model.model.layers.54.norm.weight", |
| 155 | "language_model.model.layers.54.mixer.qkv_proj.weight", |
| 156 | "language_model.model.layers.55.norm.weight", |
| 157 | "language_model.model.layers.55.mixer.down_proj.weight", |
| 158 | "visual.encoder.embeddings.patch_embedding.weight", |
| 159 | "visual.encoder.encoder.layers.0.self_attn.qkv_proj.weight", |
| 160 | "visual.projector.linear_fc1.weight", |
| 161 | "language_model.lm_head.weight", |
| 162 | ] |
| 163 | |
| 164 | assert ( |
| 165 | mapper.apply_list( |
| 166 | [ |
| 167 | "layers.0.self_attn.add_q_proj.weight", |
| 168 | "layers.0.self_attn.add_k_proj.weight", |
| 169 | "layers.0.self_attn.add_v_proj.weight", |
| 170 | "layers.0.self_attn.to_add_out.weight", |
| 171 | "layers.0.self_attn.norm_added_q.weight", |
| 172 | "layers.0.self_attn.norm_added_k.weight", |
| 173 | "layers.0.self_attn.k_norm_und_for_gen.weight", |
| 174 | "layers.0.self_attn.q_proj_moe_gen.weight", |
| 175 | "layers.0.mlp_moe_gen.up_proj.weight", |
| 176 | "norm_moe_gen.weight", |
| 177 | "proj_in.weight", |
| 178 | "proj_out.weight", |
| 179 | "audio_modality_embed", |
| 180 | "action_modality_embed", |
| 181 | ] |
| 182 | ) |
| 183 | == [] |
| 184 | ) |
| 185 | |
| 186 | |
| 187 | @pytest.mark.cpu_test |
| 188 | def test_bailing_vl_mapper_handles_module_and_parameter_names(): |
| 189 | from vllm.model_executor.models.bailing_moe_v3_vl import ( |
| 190 | BailingMoeV3VLForConditionalGeneration, |
| 191 | ) |
| 192 | |
| 193 | mapper = BailingMoeV3VLForConditionalGeneration.hf_to_vllm_mapper |
| 194 | names = { |
| 195 | "model.visual": "visual", |
| 196 | "model.visual.blocks.0.attn.qkv.weight": "visual.blocks.0.attn.qkv.weight", |
| 197 | "model.linear_proj": "linear_proj", |
| 198 | "linear_proj.0": "linear_proj.linear_fc1", |
| 199 | "model.linear_proj.0.weight": "linear_proj.linear_fc1.weight", |
| 200 | "linear_proj.2": "linear_proj.linear_fc2", |
| 201 | "linear_proj.2.weight_scale": "linear_proj.linear_fc2.weight_scale", |
| 202 | "model.linear_proj.2.bias": "linear_proj.linear_fc2.bias", |
| 203 | "linear_proj.linear_fc1.weight": "linear_proj.linear_fc1.weight", |
| 204 | "model.linear_proj.linear_fc2.weight": "linear_proj.linear_fc2.weight", |
| 205 | "model.layers.0.attention.q_proj.weight": ( |
| 206 | "language_model.model.layers.0.self_attn.q_proj.weight" |
| 207 | ), |
| 208 | "model.layers.0.mlp.experts.3.up_proj.weight": ( |
| 209 | "language_model.model.layers.0.mlp.experts.3.up_proj.weight" |
| 210 | ), |
| 211 | "lm_head": "language_model.lm_head", |
| 212 | "lm_head.weight": "language_model.lm_head.weight", |
| 213 | } |
| 214 | assert mapper.apply_list(list(names)) == list(names.values()) |
| 215 | |
| 216 | |
| 217 | def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]: |
| 218 | """Create weights from safetensors checkpoint metadata.""" |
| 219 | metadata = try_get_safetensors_metadata(repo) |
| 220 | weight_names = list(metadata.weight_map.keys()) |
| 221 | with torch.device("meta"): |
| 222 | return ((name, torch.empty(0)) for name in weight_names) |
| 223 | |
| 224 | |
| 225 | def create_dummy_base_model(repo: str, model_arch: str) -> PreTrainedModel: |
| 226 | """Create weights from a dummy meta-deserialized HF base model, with renaming.""" |
| 227 | config = AutoConfig.from_pretrained(repo) |
| 228 | with torch.device("meta"): |
| 229 | model = AutoModel.from_config(config) |
| 230 | return model |
| 231 | |
| 232 | |
| 233 | def create_dummy_model(repo: str, model_arch: str) -> PreTrainedModel: |
| 234 | """Create weights from a dummy meta deserialized hf model with name conversion.""" |
| 235 | model_cls: PreTrainedModel = getattr(transformers, model_arch) |
| 236 | config = AutoConfig.from_pretrained(repo) |
| 237 | with torch.device("meta"): |
| 238 | model = model_cls._from_config(config) |
| 239 | return model |
| 240 | |
| 241 | |
| 242 | def model_architectures_for_test() -> list[str]: |
| 243 | arch_to_test = list[str]() |
| 244 | for model_arch, info in _MULTIMODAL_EXAMPLE_MODELS.items(): |
| 245 | if not info.trust_remote_code and hasattr(transformers, model_arch): |
| 246 | model_cls: PreTrainedModel = getattr(transformers, model_arch) |
| 247 | if getattr(model_cls, "_checkpoint_conversion_mapping", None): |
| 248 | arch_to_test.append(model_arch) |
| 249 | return arch_to_test |
| 250 | |
| 251 | |
| 252 | @pytest.mark.core_model |
| 253 | @pytest.mark.parametrize("model_arch", model_architectures_for_test()) |
| 254 | def test_hf_model_weights_mapper(model_arch: str): |
| 255 | model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch) |
| 256 | model_info.check_available_online(on_fail="skip") |
| 257 | model_info.check_transformers_version(on_fail="skip") |
| 258 | |
| 259 | is_mistral_model = model_arch in [ |
| 260 | "Mistral3ForConditionalGeneration", |
| 261 | "PixtralForConditionalGeneration", |
| 262 | "VoxtralForConditionalGeneration", |
| 263 | ] |
| 264 | |
| 265 | if not is_mistral_model or model_info.tokenizer_mode == "mistral": |
| 266 | tokenizer_mode = model_info.tokenizer_mode |
| 267 | else: |
| 268 | tokenizer_mode = "hf" |
| 269 | |
| 270 | model_id = model_info.default |
| 271 | |
| 272 | model_config = ModelConfig( |
| 273 | model_id, |
| 274 | tokenizer=model_info.tokenizer or model_id, |
| 275 | tokenizer_mode=tokenizer_mode, |
| 276 | config_format="hf", |
| 277 | revision=model_info.revision, |
| 278 | trust_remote_code=model_info.trust_remote_code, |
| 279 | hf_overrides=model_info.hf_overrides, |
| 280 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 281 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 282 | enable_mm_embeds=model_info.require_embed_inputs, |
| 283 | enforce_eager=model_info.enforce_eager, |
| 284 | dtype=model_info.dtype, |
| 285 | ) |
| 286 | model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config) |
| 287 | if issubclass(model_cls, TransformersBase): |
| 288 | # Transformers backend models create their mapper during __init__ |
| 289 | # by inspecting the HF model instance. We simulate this by calling |
| 290 | # _create_hf_to_vllm_mapper with a minimal proxy object. |
| 291 | model_cls = type( |
| 292 | "ProxyModelCls", |
| 293 | (), |
| 294 | { |
| 295 | "model": create_dummy_base_model(model_id, model_arch), |
| 296 | "_maybe_apply_model_mapping": lambda self: None, |
| 297 | }, |
| 298 | )() |
| 299 | TransformersBase._create_hf_to_vllm_mapper(model_cls) |
| 300 | |
| 301 | original_weights = create_repo_dummy_weights(model_id) |
| 302 | hf_dummy_model = create_dummy_model(model_id, model_arch) |
| 303 | hf_converted_weights = hf_dummy_model.named_parameters() |
| 304 | hf_converted_buffers = hf_dummy_model.named_buffers() |
| 305 | mapper: WeightsMapper = model_cls.hf_to_vllm_mapper |
| 306 | |
| 307 | mapped_original_weights = mapper.apply(original_weights) |
| 308 | mapped_hf_converted_weights = mapper.apply(hf_converted_weights) |
| 309 | mapped_hf_converted_buffers = mapper.apply(hf_converted_buffers) |
| 310 | |
| 311 | ref_weight_names = set(map(lambda x: x[0], mapped_original_weights)) |
| 312 | weight_names = set(map(lambda x: x[0], mapped_hf_converted_weights)) |
| 313 | buffer_names = set(map(lambda x: x[0], mapped_hf_converted_buffers)) |
| 314 | |
| 315 | # Some checkpoints may have buffers, we ignore them for this test |
| 316 | ref_weight_names -= buffer_names |
| 317 | |
| 318 | # Some checkpoints include tied weights (e.g. lm_head tied to embed_tokens) in the |
| 319 | # safetensors file. In Transformers v5, named_parameters() will not include them |
| 320 | # after they are tied in the model, so the mapper will not be able to map them. |
| 321 | # We exclude them from the reference weight names for this test. |
| 322 | if isinstance(tied := getattr(hf_dummy_model, "_tied_weights_keys", None), dict): |
| 323 | config = hf_dummy_model.config |
| 324 | key = "tie_word_embeddings" |
| 325 | if getattr(config.get_text_config(), key, False) or getattr(config, key, False): |
| 326 | mapped_tied_weights = mapper.apply((k, None) for k in tied) |
| 327 | tied_weight_names = set(map(lambda x: x[0], mapped_tied_weights)) |
| 328 | ref_weight_names -= tied_weight_names |
| 329 | |
| 330 | weights_missing = ref_weight_names - weight_names |
| 331 | weights_unmapped = weight_names - ref_weight_names |
| 332 | assert not weights_missing and not weights_unmapped, ( |
| 333 | f"Following weights are not mapped correctly: {weights_unmapped}, " |
| 334 | f"Missing expected weights: {weights_missing}." |
| 335 | ) |