A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Refactor Molmo weights loading to use AutoWeightsLoader (#10771)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 30, 2024 at 12:19 UTC
16ee07f22ade57eb882b3c16ad3a6944635996df
1 file changed
+111
-102
vllm/model_executor/models/molmo.py
+111
-102
@@ -3,7 +3,7 @@ import re
3
from array import array
4
from dataclasses import dataclass
5
from functools import lru_cache, partial
6
-from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict
6
+from typing import Iterable, List, Mapping, Optional, Set, Tuple, TypedDict
7
8
import torch
9
from einops import rearrange
@@ -44,7 +44,8 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44
from vllm.transformers_utils.processor import get_processor
45
46
from .interfaces import SupportsMultiModal, SupportsPP
47
-from .utils import (get_vit_attn_backend,
47
+from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
48
+ is_pp_missing_parameter,
49
make_empty_intermediate_tensors_factory, make_layers,
50
maybe_prefix)
51
@@ -720,6 +721,42 @@ class MolmoVisionBackbone(nn.Module):
721
# image_features: (batch_size, num_image, num_patch, d_model)
722
return image_features
723
724
+ def load_weights(self, weights: Iterable[Tuple[str,
725
+ torch.Tensor]]) -> Set[str]:
726
+ stacked_params_mapping = [
727
+ # (param_name, shard_name, shard_id)
728
+ ("gate_up_proj", "gate_proj", 0),
729
+ ("gate_up_proj", "up_proj", 1),
730
+ ]
731
+ params_dict = dict(self.named_parameters())
732
+ loaded_params: Set[str] = set()
733
+
734
+ for name, loaded_weight in weights:
735
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
736
+ if weight_name not in name:
737
+ continue
738
+ name = name.replace(weight_name, param_name)
739
+ # Skip loading extra bias for GPTQ models.
740
+ if name.endswith(".bias") and name not in params_dict:
741
+ continue
742
+ if is_pp_missing_parameter(name, self):
743
+ continue
744
+ param = params_dict[name]
745
+ weight_loader = param.weight_loader
746
+ weight_loader(param, loaded_weight, shard_id)
747
+ break
748
+ else:
749
+ if name.endswith(".bias") and name not in params_dict:
750
+ continue
751
+ if is_pp_missing_parameter(name, self):
752
+ continue
753
+ param = params_dict[name]
754
+ weight_loader = getattr(param, "weight_loader",
755
+ default_weight_loader)
756
+ weight_loader(param, loaded_weight)
757
+ loaded_params.add(name)
758
+ return loaded_params
759
+
760
761
@support_torch_compile
762
class MolmoModel(nn.Module):
@@ -804,6 +841,28 @@ class MolmoModel(nn.Module):
841
hidden_states = self.norm(hidden_states)
842
return hidden_states
843
844
+ def load_weights(self, weights: Iterable[Tuple[str,
845
+ torch.Tensor]]) -> Set[str]:
846
+ params_dict = dict(self.named_parameters())
847
+ loaded_params: Set[str] = set()
848
+
849
+ for name, loaded_weight in weights:
850
+ if "gate_up_proj" in name:
851
+ up_proj, gate_proj = loaded_weight.chunk(2, dim=0)
852
+ loaded_weight = torch.cat([gate_proj, up_proj], dim=0)
853
+
854
+ if name.endswith(".bias") and name not in params_dict:
855
+ continue
856
+ if is_pp_missing_parameter(name, self):
857
+ continue
858
+
859
+ param = params_dict[name]
860
+ weight_loader = getattr(param, "weight_loader",
861
+ default_weight_loader)
862
+ weight_loader(param, loaded_weight)
863
+ loaded_params.add(name)
864
+ return loaded_params
865
+
866
867
cached_get_processor = lru_cache(get_processor)
868
@@ -1200,103 +1259,53 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1259
return next_tokens
1260
1261
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1203
-
1204
- params_mapping = [
1205
- ("model.transformer.ln_f.weight", "model.norm.weight"),
1206
- ("attn_out", "self_attn.o_proj"),
1207
- ("att_proj", "self_attn.qkv_proj"),
1208
- ("q_norm", "self_attn.q_norm"),
1209
- ("k_norm", "self_attn.k_norm"),
1210
- ("attn_norm", "input_layernorm"),
1211
- ("ff_norm", "post_attention_layernorm"),
1212
- ]
1213
-
1214
- params_dict = dict(self.named_parameters(remove_duplicate=False))
1215
-
1216
- embedding_weight = dict()
1217
- projector_weight = dict()
1218
- for name, loaded_weight in weights:
1219
- if "rotary_emb.inv_freq" in name:
1220
- continue
1221
- if self.config.tie_word_embeddings and "lm_head.weight" in name:
1222
- continue
1223
-
1224
- if "wte.embedding" in name:
1225
- embedding_weight["embedding"] = loaded_weight
1226
- continue
1227
-
1228
- if "wte.new_embedding" in name:
1229
- embedding_weight["new_embedding"] = loaded_weight
1230
- continue
1231
-
1232
- if "vision_backbone" in name:
1233
- if name.startswith("model"):
1234
- name = name[len("model."):]
1235
- if 'image_projector' in name:
1236
- if 'w1' in name:
1237
- projector_weight['gate_proj'] = loaded_weight
1238
- elif 'w3' in name:
1239
- projector_weight['up_proj'] = loaded_weight
1240
- elif 'w2' in name:
1241
- projector_weight['down_proj'] = loaded_weight
1242
- else:
1243
- raise ValueError(
1244
- f"Unexpected projector weight: {name}")
1245
- continue
1246
- else:
1247
- if "transformer.blocks" in name:
1248
- name = name.replace("transformer.blocks", "layers")
1249
-
1250
- if "ff_proj" in name:
1251
- name = name.replace("ff_proj", "mlp.gate_up_proj")
1252
- assert 'weight' in name
1253
- up_weight, gate_weight = loaded_weight.chunk(2, dim=0)
1254
- loaded_weight = torch.cat([gate_weight, up_weight], dim=0)
1255
-
1256
- elif "ff_out" in name:
1257
- if "layers" in name:
1258
- name = name.replace("ff_out", "mlp.down_proj")
1259
- else:
1260
- # lm head
1261
- name = name.replace("model.transformer.ff_out",
1262
- "lm_head")
1263
-
1264
- else:
1265
- for (param_name, weight_name) in params_mapping:
1266
- if param_name in name:
1267
- name = name.replace(param_name, weight_name)
1268
- break
1269
-
1270
- try:
1271
- # Skip loading extra bias for GPTQ models.
1272
- if name.endswith(".bias") and name not in params_dict:
1273
- continue
1274
- param = params_dict[name]
1275
- except KeyError:
1276
- raise ValueError(f"Unexpected weight: {name}") from None
1277
-
1278
- weight_loader = getattr(param, "weight_loader",
1279
- default_weight_loader)
1280
- weight_loader(param, loaded_weight)
1281
-
1282
- gate_up_proj_weight = torch.cat(
1283
- [projector_weight["gate_proj"], projector_weight["up_proj"]],
1284
- dim=0)
1285
- name = "vision_backbone.image_projector.gate_up_proj.weight"
1286
- param = params_dict[name]
1287
- weight_loader = getattr(param, "weight_loader", default_weight_loader)
1288
- weight_loader(param, gate_up_proj_weight)
1289
-
1290
- down_proj_weight = projector_weight["down_proj"]
1291
- name = "vision_backbone.image_projector.down_proj.weight"
1292
- param = params_dict[name]
1293
- weight_loader = getattr(param, "weight_loader", default_weight_loader)
1294
- weight_loader(param, down_proj_weight)
1295
-
1296
- embedding_weight = torch.cat(
1297
- [embedding_weight["embedding"], embedding_weight["new_embedding"]],
1298
- dim=0)
1299
- name = "model.embed_tokens.weight"
1300
- param = params_dict[name]
1301
- weight_loader = getattr(param, "weight_loader", default_weight_loader)
1302
- weight_loader(param, embedding_weight)
1262
+ hf_to_vllm_mapper = WeightsMapper(
1263
+ orig_to_new_substr={
1264
+ # vision backbone mapping
1265
+ "image_projector.w1.": "image_projector.gate_proj.",
1266
+ "image_projector.w3.": "image_projector.up_proj.",
1267
+ "image_projector.w2.": "image_projector.down_proj.",
1268
+ # language backbone mapping
1269
+ "att_proj": "self_attn.qkv_proj",
1270
+ "attn_out": "self_attn.o_proj",
1271
+ "q_norm": "self_attn.q_norm",
1272
+ "k_norm": "self_attn.k_norm",
1273
+ "ff_proj": "mlp.gate_up_proj",
1274
+ "ff_out": "mlp.down_proj",
1275
+ "attn_norm": "input_layernorm",
1276
+ "ff_norm": "post_attention_layernorm",
1277
+ },
1278
+ orig_to_new_prefix={
1279
+ # vision backbone mapping
1280
+ "model.vision_backbone.": "vision_backbone.",
1281
+ # language backbone mapping
1282
+ "model.transformer.blocks.": "model.layers.",
1283
+ "model.transformer.ln_f.": "model.norm.",
1284
+ # lm_head is renamed to model.transformer.mlp.down_proj firstly,
1285
+ # we need to run a second renaming for it
1286
+ "model.transformer.mlp.down_proj.": "lm_head.",
1287
+ },
1288
+ )
1289
+ loader = AutoWeightsLoader(self)
1290
+ weights = _get_weights_with_merged_embedding(weights)
1291
+ return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
1292
+
1293
+
1294
+def _get_weights_with_merged_embedding(
1295
+ weights: Iterable[Tuple[str, torch.Tensor]]
1296
+) -> Iterable[Tuple[str, torch.Tensor]]:
1297
+ embedding_weights = {}
1298
+ for name, weight in weights:
1299
+ if "wte.embedding" in name:
1300
+ embedding_weights["embedding"] = weight
1301
+ elif "wte.new_embedding" in name:
1302
+ embedding_weights["new_embedding"] = weight
1303
+ else:
1304
+ yield (name, weight)
1305
+ # this is compatible with most of quantization,
1306
+ # because they won't quantize embed_tokens
1307
+ embedding_weights = torch.cat(
1308
+ [embedding_weights["embedding"], embedding_weights["new_embedding"]],
1309
+ dim=0,
1310
+ )
1311
+ yield ("model.embed_tokens.weight", embedding_weights)