@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16ee07f22a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Refactor Molmo weights loading to use AutoWeightsLoader (#10771)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 30, 2024 at 12:19 UTC 16ee07f22ade57eb882b3c16ad3a6944635996df
1 file changed +111 -102
vllm/model_executor/models/molmo.py
+111 -102
@@ -3,7 +3,7 @@ import re
3 from array import array
4 from dataclasses import dataclass
5 from functools import lru_cache, partial
6 -from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict
6 +from typing import Iterable, List, Mapping, Optional, Set, Tuple, TypedDict
7
8 import torch
9 from einops import rearrange
@@ -44,7 +44,8 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44 from vllm.transformers_utils.processor import get_processor
45
46 from .interfaces import SupportsMultiModal, SupportsPP
47 -from .utils import (get_vit_attn_backend,
47 +from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
48 + is_pp_missing_parameter,
49 make_empty_intermediate_tensors_factory, make_layers,
50 maybe_prefix)
51
@@ -720,6 +721,42 @@ class MolmoVisionBackbone(nn.Module):
721 # image_features: (batch_size, num_image, num_patch, d_model)
722 return image_features
723
724 + def load_weights(self, weights: Iterable[Tuple[str,
725 + torch.Tensor]]) -> Set[str]:
726 + stacked_params_mapping = [
727 + # (param_name, shard_name, shard_id)
728 + ("gate_up_proj", "gate_proj", 0),
729 + ("gate_up_proj", "up_proj", 1),
730 + ]
731 + params_dict = dict(self.named_parameters())
732 + loaded_params: Set[str] = set()
733 +
734 + for name, loaded_weight in weights:
735 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
736 + if weight_name not in name:
737 + continue
738 + name = name.replace(weight_name, param_name)
739 + # Skip loading extra bias for GPTQ models.
740 + if name.endswith(".bias") and name not in params_dict:
741 + continue
742 + if is_pp_missing_parameter(name, self):
743 + continue
744 + param = params_dict[name]
745 + weight_loader = param.weight_loader
746 + weight_loader(param, loaded_weight, shard_id)
747 + break
748 + else:
749 + if name.endswith(".bias") and name not in params_dict:
750 + continue
751 + if is_pp_missing_parameter(name, self):
752 + continue
753 + param = params_dict[name]
754 + weight_loader = getattr(param, "weight_loader",
755 + default_weight_loader)
756 + weight_loader(param, loaded_weight)
757 + loaded_params.add(name)
758 + return loaded_params
759 +
760
761 @support_torch_compile
762 class MolmoModel(nn.Module):
@@ -804,6 +841,28 @@ class MolmoModel(nn.Module):
841 hidden_states = self.norm(hidden_states)
842 return hidden_states
843
844 + def load_weights(self, weights: Iterable[Tuple[str,
845 + torch.Tensor]]) -> Set[str]:
846 + params_dict = dict(self.named_parameters())
847 + loaded_params: Set[str] = set()
848 +
849 + for name, loaded_weight in weights:
850 + if "gate_up_proj" in name:
851 + up_proj, gate_proj = loaded_weight.chunk(2, dim=0)
852 + loaded_weight = torch.cat([gate_proj, up_proj], dim=0)
853 +
854 + if name.endswith(".bias") and name not in params_dict:
855 + continue
856 + if is_pp_missing_parameter(name, self):
857 + continue
858 +
859 + param = params_dict[name]
860 + weight_loader = getattr(param, "weight_loader",
861 + default_weight_loader)
862 + weight_loader(param, loaded_weight)
863 + loaded_params.add(name)
864 + return loaded_params
865 +
866
867 cached_get_processor = lru_cache(get_processor)
868
@@ -1200,103 +1259,53 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1259 return next_tokens
1260
1261 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1203 -
1204 - params_mapping = [
1205 - ("model.transformer.ln_f.weight", "model.norm.weight"),
1206 - ("attn_out", "self_attn.o_proj"),
1207 - ("att_proj", "self_attn.qkv_proj"),
1208 - ("q_norm", "self_attn.q_norm"),
1209 - ("k_norm", "self_attn.k_norm"),
1210 - ("attn_norm", "input_layernorm"),
1211 - ("ff_norm", "post_attention_layernorm"),
1212 - ]
1213 -
1214 - params_dict = dict(self.named_parameters(remove_duplicate=False))
1215 -
1216 - embedding_weight = dict()
1217 - projector_weight = dict()
1218 - for name, loaded_weight in weights:
1219 - if "rotary_emb.inv_freq" in name:
1220 - continue
1221 - if self.config.tie_word_embeddings and "lm_head.weight" in name:
1222 - continue
1223 -
1224 - if "wte.embedding" in name:
1225 - embedding_weight["embedding"] = loaded_weight
1226 - continue
1227 -
1228 - if "wte.new_embedding" in name:
1229 - embedding_weight["new_embedding"] = loaded_weight
1230 - continue
1231 -
1232 - if "vision_backbone" in name:
1233 - if name.startswith("model"):
1234 - name = name[len("model."):]
1235 - if 'image_projector' in name:
1236 - if 'w1' in name:
1237 - projector_weight['gate_proj'] = loaded_weight
1238 - elif 'w3' in name:
1239 - projector_weight['up_proj'] = loaded_weight
1240 - elif 'w2' in name:
1241 - projector_weight['down_proj'] = loaded_weight
1242 - else:
1243 - raise ValueError(
1244 - f"Unexpected projector weight: {name}")
1245 - continue
1246 - else:
1247 - if "transformer.blocks" in name:
1248 - name = name.replace("transformer.blocks", "layers")
1249 -
1250 - if "ff_proj" in name:
1251 - name = name.replace("ff_proj", "mlp.gate_up_proj")
1252 - assert 'weight' in name
1253 - up_weight, gate_weight = loaded_weight.chunk(2, dim=0)
1254 - loaded_weight = torch.cat([gate_weight, up_weight], dim=0)
1255 -
1256 - elif "ff_out" in name:
1257 - if "layers" in name:
1258 - name = name.replace("ff_out", "mlp.down_proj")
1259 - else:
1260 - # lm head
1261 - name = name.replace("model.transformer.ff_out",
1262 - "lm_head")
1263 -
1264 - else:
1265 - for (param_name, weight_name) in params_mapping:
1266 - if param_name in name:
1267 - name = name.replace(param_name, weight_name)
1268 - break
1269 -
1270 - try:
1271 - # Skip loading extra bias for GPTQ models.
1272 - if name.endswith(".bias") and name not in params_dict:
1273 - continue
1274 - param = params_dict[name]
1275 - except KeyError:
1276 - raise ValueError(f"Unexpected weight: {name}") from None
1277 -
1278 - weight_loader = getattr(param, "weight_loader",
1279 - default_weight_loader)
1280 - weight_loader(param, loaded_weight)
1281 -
1282 - gate_up_proj_weight = torch.cat(
1283 - [projector_weight["gate_proj"], projector_weight["up_proj"]],
1284 - dim=0)
1285 - name = "vision_backbone.image_projector.gate_up_proj.weight"
1286 - param = params_dict[name]
1287 - weight_loader = getattr(param, "weight_loader", default_weight_loader)
1288 - weight_loader(param, gate_up_proj_weight)
1289 -
1290 - down_proj_weight = projector_weight["down_proj"]
1291 - name = "vision_backbone.image_projector.down_proj.weight"
1292 - param = params_dict[name]
1293 - weight_loader = getattr(param, "weight_loader", default_weight_loader)
1294 - weight_loader(param, down_proj_weight)
1295 -
1296 - embedding_weight = torch.cat(
1297 - [embedding_weight["embedding"], embedding_weight["new_embedding"]],
1298 - dim=0)
1299 - name = "model.embed_tokens.weight"
1300 - param = params_dict[name]
1301 - weight_loader = getattr(param, "weight_loader", default_weight_loader)
1302 - weight_loader(param, embedding_weight)
1262 + hf_to_vllm_mapper = WeightsMapper(
1263 + orig_to_new_substr={
1264 + # vision backbone mapping
1265 + "image_projector.w1.": "image_projector.gate_proj.",
1266 + "image_projector.w3.": "image_projector.up_proj.",
1267 + "image_projector.w2.": "image_projector.down_proj.",
1268 + # language backbone mapping
1269 + "att_proj": "self_attn.qkv_proj",
1270 + "attn_out": "self_attn.o_proj",
1271 + "q_norm": "self_attn.q_norm",
1272 + "k_norm": "self_attn.k_norm",
1273 + "ff_proj": "mlp.gate_up_proj",
1274 + "ff_out": "mlp.down_proj",
1275 + "attn_norm": "input_layernorm",
1276 + "ff_norm": "post_attention_layernorm",
1277 + },
1278 + orig_to_new_prefix={
1279 + # vision backbone mapping
1280 + "model.vision_backbone.": "vision_backbone.",
1281 + # language backbone mapping
1282 + "model.transformer.blocks.": "model.layers.",
1283 + "model.transformer.ln_f.": "model.norm.",
1284 + # lm_head is renamed to model.transformer.mlp.down_proj firstly,
1285 + # we need to run a second renaming for it
1286 + "model.transformer.mlp.down_proj.": "lm_head.",
1287 + },
1288 + )
1289 + loader = AutoWeightsLoader(self)
1290 + weights = _get_weights_with_merged_embedding(weights)
1291 + return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
1292 +
1293 +
1294 +def _get_weights_with_merged_embedding(
1295 + weights: Iterable[Tuple[str, torch.Tensor]]
1296 +) -> Iterable[Tuple[str, torch.Tensor]]:
1297 + embedding_weights = {}
1298 + for name, weight in weights:
1299 + if "wte.embedding" in name:
1300 + embedding_weights["embedding"] = weight
1301 + elif "wte.new_embedding" in name:
1302 + embedding_weights["new_embedding"] = weight
1303 + else:
1304 + yield (name, weight)
1305 + # this is compatible with most of quantization,
1306 + # because they won't quantize embed_tokens
1307 + embedding_weights = torch.cat(
1308 + [embedding_weights["embedding"], embedding_weights["new_embedding"]],
1309 + dim=0,
1310 + )
1311 + yield ("model.embed_tokens.weight", embedding_weights)