A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Move weights mapper (#11443)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 24, 2024 at 21:05 UTC
196c34b0acdd19014feb6c065c324036407f3b36
8 files changed
+74
-68
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py
+3
-2
@@ -13,6 +13,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
13
14
15
class MyGemma2Embedding(nn.Module):
16
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
17
18
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
19
super().__init__()
@@ -62,8 +63,8 @@ class MyGemma2Embedding(nn.Module):
63
return self._pooler(hidden_states, pooling_metadata)
64
65
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
65
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
66
- weights = hf_to_vllm_mapper.apply(weights)
66
+
67
+ weights = self.hf_to_vllm_mapper.apply(weights)
68
weights = ((name, data) for name, data in weights
69
if not name.startswith("lm_head."))
70
return self.model.load_weights(weights)
vllm/model_executor/models/aria.py
+10
-10
@@ -521,6 +521,15 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
521
This model combines a vision tower, a multi-modal projector, and a language
522
model to perform tasks that involve both image and text inputs.
523
"""
524
+ hf_to_vllm_mapper = WeightsMapper(
525
+ orig_to_new_prefix={
526
+ "language_model.model": "language_model",
527
+ "language_model.lm_head": "lm_head",
528
+ },
529
+ orig_to_new_suffix={
530
+ "router.weight": "router_weight",
531
+ },
532
+ )
533
534
def __init__(
535
self,
@@ -662,15 +671,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
671
return next_tokens
672
673
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
665
- hf_to_vllm_mapper = WeightsMapper(
666
- orig_to_new_prefix={
667
- "language_model.model": "language_model",
668
- "language_model.lm_head": "lm_head",
669
- },
670
- orig_to_new_suffix={
671
- "router.weight": "router_weight",
672
- },
673
- )
674
675
loader = AutoWeightsLoader(self)
676
- loader.load_weights(weights, mapper=hf_to_vllm_mapper)
676
+ loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/bert.py
+2
-2
@@ -409,6 +409,7 @@ class BertEmbeddingModel(nn.Module):
409
model: An instance of BertModel used for forward operations.
410
_pooler: An instance of Pooler used for pooling operations.
411
"""
412
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
413
414
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
415
super().__init__()
@@ -441,8 +442,7 @@ class BertEmbeddingModel(nn.Module):
442
return self._pooler(hidden_states, pooling_metadata)
443
444
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
444
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
445
- weights = hf_to_vllm_mapper.apply(weights)
445
+ weights = self.hf_to_vllm_mapper.apply(weights)
446
weights = ((name, data) for name, data in weights
447
if not name.startswith("lm_head."))
448
self.model.load_weights(weights)
vllm/model_executor/models/molmo.py
+30
-28
@@ -1123,6 +1123,34 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1123
@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1124
class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1125
1126
+ hf_to_vllm_mapper = WeightsMapper(
1127
+ orig_to_new_substr={
1128
+ # vision backbone mapping
1129
+ "image_projector.w1.": "image_projector.gate_proj.",
1130
+ "image_projector.w3.": "image_projector.up_proj.",
1131
+ "image_projector.w2.": "image_projector.down_proj.",
1132
+ # language backbone mapping
1133
+ "att_proj": "self_attn.qkv_proj",
1134
+ "attn_out": "self_attn.o_proj",
1135
+ "q_norm": "self_attn.q_norm",
1136
+ "k_norm": "self_attn.k_norm",
1137
+ "ff_proj": "mlp.gate_up_proj",
1138
+ "ff_out": "mlp.down_proj",
1139
+ "attn_norm": "input_layernorm",
1140
+ "ff_norm": "post_attention_layernorm",
1141
+ },
1142
+ orig_to_new_prefix={
1143
+ # vision backbone mapping
1144
+ "model.vision_backbone.": "vision_backbone.",
1145
+ # language backbone mapping
1146
+ "model.transformer.blocks.": "model.layers.",
1147
+ "model.transformer.ln_f.": "model.norm.",
1148
+ # lm_head is renamed to model.transformer.mlp.down_proj firstly,
1149
+ # we need to run a second renaming for it
1150
+ "model.transformer.mlp.down_proj.": "lm_head.",
1151
+ },
1152
+ )
1153
+
1154
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1155
super().__init__()
1156
config = vllm_config.model_config.hf_config
@@ -1298,36 +1326,10 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1326
return next_tokens
1327
1328
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1301
- hf_to_vllm_mapper = WeightsMapper(
1302
- orig_to_new_substr={
1303
- # vision backbone mapping
1304
- "image_projector.w1.": "image_projector.gate_proj.",
1305
- "image_projector.w3.": "image_projector.up_proj.",
1306
- "image_projector.w2.": "image_projector.down_proj.",
1307
- # language backbone mapping
1308
- "att_proj": "self_attn.qkv_proj",
1309
- "attn_out": "self_attn.o_proj",
1310
- "q_norm": "self_attn.q_norm",
1311
- "k_norm": "self_attn.k_norm",
1312
- "ff_proj": "mlp.gate_up_proj",
1313
- "ff_out": "mlp.down_proj",
1314
- "attn_norm": "input_layernorm",
1315
- "ff_norm": "post_attention_layernorm",
1316
- },
1317
- orig_to_new_prefix={
1318
- # vision backbone mapping
1319
- "model.vision_backbone.": "vision_backbone.",
1320
- # language backbone mapping
1321
- "model.transformer.blocks.": "model.layers.",
1322
- "model.transformer.ln_f.": "model.norm.",
1323
- # lm_head is renamed to model.transformer.mlp.down_proj firstly,
1324
- # we need to run a second renaming for it
1325
- "model.transformer.mlp.down_proj.": "lm_head.",
1326
- },
1327
- )
1329
+
1330
loader = AutoWeightsLoader(self)
1331
weights = _get_weights_with_merged_embedding(weights)
1330
- return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
1332
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1333
1334
1335
def _get_weights_with_merged_embedding(
vllm/model_executor/models/phi3v.py
+8
-8
@@ -408,6 +408,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
408
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
409
@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
410
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
411
+ hf_to_vllm_mapper = WeightsMapper(
412
+ orig_to_new_prefix={
413
+ "model.vision_embed_tokens.wte": "embed_tokens",
414
+ "model.vision_embed_tokens.": "vision_embed_tokens.",
415
+ "lm_head.": "language_model.lm_head.",
416
+ "model.": "language_model.model.",
417
+ })
418
419
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
420
super().__init__()
@@ -616,17 +623,10 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
623
624
def load_weights(self, weights: Iterable[Tuple[str,
625
torch.Tensor]]) -> Set[str]:
619
- hf_to_vllm_mapper = WeightsMapper(
620
- orig_to_new_prefix={
621
- "model.vision_embed_tokens.wte": "embed_tokens",
622
- "model.vision_embed_tokens.": "vision_embed_tokens.",
623
- "lm_head.": "language_model.lm_head.",
624
- "model.": "language_model.model.",
625
- })
626
627
loader = AutoWeightsLoader(self)
628
autoloaded_weights = loader.load_weights(weights,
629
- mapper=hf_to_vllm_mapper)
629
+ mapper=self.hf_to_vllm_mapper)
630
631
# The HF config doesn't specify whether these are tied,
632
# so we detect it this way
vllm/model_executor/models/qwen2.py
+3
-2
@@ -529,6 +529,8 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
529
embedding_modules = {}
530
embedding_padding_modules = []
531
532
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
533
+
534
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
535
super().__init__()
536
config = vllm_config.model_config.hf_config
@@ -577,8 +579,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
579
return self._pooler(hidden_states, pooling_metadata)
580
581
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
580
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
581
- weights = hf_to_vllm_mapper.apply(weights)
582
+ weights = self.hf_to_vllm_mapper.apply(weights)
583
weights = ((name, data) for name, data in weights
584
if not name.startswith("lm_head."))
585
self.model.load_weights(weights)
vllm/model_executor/models/telechat2.py
+14
-13
@@ -31,6 +31,19 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
31
32
class TeleChat2Model(LlamaModel):
33
34
+ hf_to_vllm_mapper = WeightsMapper(
35
+ orig_to_new_prefix={
36
+ "transformer.": "model.",
37
+ },
38
+ orig_to_new_substr={
39
+ ".h.": ".layers.",
40
+ ".self_attention.": ".self_attn.",
41
+ ".word_embeddings.": ".embed_tokens.",
42
+ ".dense.": ".o_proj.",
43
+ ".ln_f.": ".norm.",
44
+ },
45
+ )
46
+
47
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
48
# 1. Initialize the LlamaModel with bias
49
vllm_config.model_config.hf_config.bias = True
@@ -111,21 +124,9 @@ class TeleChat2ForCausalLM(LlamaForCausalLM):
124
def load_weights(self, weights: Iterable[Tuple[str,
125
torch.Tensor]]) -> Set[str]:
126
114
- hf_to_vllm_mapper = WeightsMapper(
115
- orig_to_new_prefix={
116
- "transformer.": "model.",
117
- },
118
- orig_to_new_substr={
119
- ".h.": ".layers.",
120
- ".self_attention.": ".self_attn.",
121
- ".word_embeddings.": ".embed_tokens.",
122
- ".dense.": ".o_proj.",
123
- ".ln_f.": ".norm.",
124
- },
125
- )
127
loader = AutoWeightsLoader(
128
self,
129
skip_prefixes=(["lm_head."]
130
if self.config.tie_word_embeddings else None),
131
)
131
- return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
132
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/ultravox.py
+4
-3
@@ -302,6 +302,9 @@ class ModifiedWhisperEncoder(WhisperEncoder):
302
@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor)
303
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
304
305
+ hf_to_vllm_mapper = WeightsMapper(
306
+ orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
307
+
308
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
309
super().__init__()
310
config = vllm_config.model_config.hf_config
@@ -494,9 +497,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
497
498
def load_weights(self, weights: Iterable[Tuple[str,
499
torch.Tensor]]) -> Set[str]:
497
- hf_to_vllm_mapper = WeightsMapper(
498
- orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
500
501
loader = AutoWeightsLoader(self,
502
ignore_unexpected_prefixes=["audio_tower."])
502
- return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
503
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)