@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 196c34b0ac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Move weights mapper (#11443)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 24, 2024 at 21:05 UTC 196c34b0acdd19014feb6c065c324036407f3b36
8 files changed +74 -68
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py
+3 -2
@@ -13,6 +13,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
13
14
15 class MyGemma2Embedding(nn.Module):
16 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
17
18 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
19 super().__init__()
@@ -62,8 +63,8 @@ class MyGemma2Embedding(nn.Module):
63 return self._pooler(hidden_states, pooling_metadata)
64
65 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
65 - hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
66 - weights = hf_to_vllm_mapper.apply(weights)
66 +
67 + weights = self.hf_to_vllm_mapper.apply(weights)
68 weights = ((name, data) for name, data in weights
69 if not name.startswith("lm_head."))
70 return self.model.load_weights(weights)
vllm/model_executor/models/aria.py
+10 -10
@@ -521,6 +521,15 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
521 This model combines a vision tower, a multi-modal projector, and a language
522 model to perform tasks that involve both image and text inputs.
523 """
524 + hf_to_vllm_mapper = WeightsMapper(
525 + orig_to_new_prefix={
526 + "language_model.model": "language_model",
527 + "language_model.lm_head": "lm_head",
528 + },
529 + orig_to_new_suffix={
530 + "router.weight": "router_weight",
531 + },
532 + )
533
534 def __init__(
535 self,
@@ -662,15 +671,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
671 return next_tokens
672
673 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
665 - hf_to_vllm_mapper = WeightsMapper(
666 - orig_to_new_prefix={
667 - "language_model.model": "language_model",
668 - "language_model.lm_head": "lm_head",
669 - },
670 - orig_to_new_suffix={
671 - "router.weight": "router_weight",
672 - },
673 - )
674
675 loader = AutoWeightsLoader(self)
676 - loader.load_weights(weights, mapper=hf_to_vllm_mapper)
676 + loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/bert.py
+2 -2
@@ -409,6 +409,7 @@ class BertEmbeddingModel(nn.Module):
409 model: An instance of BertModel used for forward operations.
410 _pooler: An instance of Pooler used for pooling operations.
411 """
412 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
413
414 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
415 super().__init__()
@@ -441,8 +442,7 @@ class BertEmbeddingModel(nn.Module):
442 return self._pooler(hidden_states, pooling_metadata)
443
444 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
444 - hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
445 - weights = hf_to_vllm_mapper.apply(weights)
445 + weights = self.hf_to_vllm_mapper.apply(weights)
446 weights = ((name, data) for name, data in weights
447 if not name.startswith("lm_head."))
448 self.model.load_weights(weights)
vllm/model_executor/models/molmo.py
+30 -28
@@ -1123,6 +1123,34 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1123 @INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1124 class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1125
1126 + hf_to_vllm_mapper = WeightsMapper(
1127 + orig_to_new_substr={
1128 + # vision backbone mapping
1129 + "image_projector.w1.": "image_projector.gate_proj.",
1130 + "image_projector.w3.": "image_projector.up_proj.",
1131 + "image_projector.w2.": "image_projector.down_proj.",
1132 + # language backbone mapping
1133 + "att_proj": "self_attn.qkv_proj",
1134 + "attn_out": "self_attn.o_proj",
1135 + "q_norm": "self_attn.q_norm",
1136 + "k_norm": "self_attn.k_norm",
1137 + "ff_proj": "mlp.gate_up_proj",
1138 + "ff_out": "mlp.down_proj",
1139 + "attn_norm": "input_layernorm",
1140 + "ff_norm": "post_attention_layernorm",
1141 + },
1142 + orig_to_new_prefix={
1143 + # vision backbone mapping
1144 + "model.vision_backbone.": "vision_backbone.",
1145 + # language backbone mapping
1146 + "model.transformer.blocks.": "model.layers.",
1147 + "model.transformer.ln_f.": "model.norm.",
1148 + # lm_head is renamed to model.transformer.mlp.down_proj firstly,
1149 + # we need to run a second renaming for it
1150 + "model.transformer.mlp.down_proj.": "lm_head.",
1151 + },
1152 + )
1153 +
1154 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1155 super().__init__()
1156 config = vllm_config.model_config.hf_config
@@ -1298,36 +1326,10 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1326 return next_tokens
1327
1328 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1301 - hf_to_vllm_mapper = WeightsMapper(
1302 - orig_to_new_substr={
1303 - # vision backbone mapping
1304 - "image_projector.w1.": "image_projector.gate_proj.",
1305 - "image_projector.w3.": "image_projector.up_proj.",
1306 - "image_projector.w2.": "image_projector.down_proj.",
1307 - # language backbone mapping
1308 - "att_proj": "self_attn.qkv_proj",
1309 - "attn_out": "self_attn.o_proj",
1310 - "q_norm": "self_attn.q_norm",
1311 - "k_norm": "self_attn.k_norm",
1312 - "ff_proj": "mlp.gate_up_proj",
1313 - "ff_out": "mlp.down_proj",
1314 - "attn_norm": "input_layernorm",
1315 - "ff_norm": "post_attention_layernorm",
1316 - },
1317 - orig_to_new_prefix={
1318 - # vision backbone mapping
1319 - "model.vision_backbone.": "vision_backbone.",
1320 - # language backbone mapping
1321 - "model.transformer.blocks.": "model.layers.",
1322 - "model.transformer.ln_f.": "model.norm.",
1323 - # lm_head is renamed to model.transformer.mlp.down_proj firstly,
1324 - # we need to run a second renaming for it
1325 - "model.transformer.mlp.down_proj.": "lm_head.",
1326 - },
1327 - )
1329 +
1330 loader = AutoWeightsLoader(self)
1331 weights = _get_weights_with_merged_embedding(weights)
1330 - return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
1332 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1333
1334
1335 def _get_weights_with_merged_embedding(
vllm/model_executor/models/phi3v.py
+8 -8
@@ -408,6 +408,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
408 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
409 @MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
410 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
411 + hf_to_vllm_mapper = WeightsMapper(
412 + orig_to_new_prefix={
413 + "model.vision_embed_tokens.wte": "embed_tokens",
414 + "model.vision_embed_tokens.": "vision_embed_tokens.",
415 + "lm_head.": "language_model.lm_head.",
416 + "model.": "language_model.model.",
417 + })
418
419 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
420 super().__init__()
@@ -616,17 +623,10 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
623
624 def load_weights(self, weights: Iterable[Tuple[str,
625 torch.Tensor]]) -> Set[str]:
619 - hf_to_vllm_mapper = WeightsMapper(
620 - orig_to_new_prefix={
621 - "model.vision_embed_tokens.wte": "embed_tokens",
622 - "model.vision_embed_tokens.": "vision_embed_tokens.",
623 - "lm_head.": "language_model.lm_head.",
624 - "model.": "language_model.model.",
625 - })
626
627 loader = AutoWeightsLoader(self)
628 autoloaded_weights = loader.load_weights(weights,
629 - mapper=hf_to_vllm_mapper)
629 + mapper=self.hf_to_vllm_mapper)
630
631 # The HF config doesn't specify whether these are tied,
632 # so we detect it this way
vllm/model_executor/models/qwen2.py
+3 -2
@@ -529,6 +529,8 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
529 embedding_modules = {}
530 embedding_padding_modules = []
531
532 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
533 +
534 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
535 super().__init__()
536 config = vllm_config.model_config.hf_config
@@ -577,8 +579,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
579 return self._pooler(hidden_states, pooling_metadata)
580
581 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
580 - hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
581 - weights = hf_to_vllm_mapper.apply(weights)
582 + weights = self.hf_to_vllm_mapper.apply(weights)
583 weights = ((name, data) for name, data in weights
584 if not name.startswith("lm_head."))
585 self.model.load_weights(weights)
vllm/model_executor/models/telechat2.py
+14 -13
@@ -31,6 +31,19 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
31
32 class TeleChat2Model(LlamaModel):
33
34 + hf_to_vllm_mapper = WeightsMapper(
35 + orig_to_new_prefix={
36 + "transformer.": "model.",
37 + },
38 + orig_to_new_substr={
39 + ".h.": ".layers.",
40 + ".self_attention.": ".self_attn.",
41 + ".word_embeddings.": ".embed_tokens.",
42 + ".dense.": ".o_proj.",
43 + ".ln_f.": ".norm.",
44 + },
45 + )
46 +
47 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
48 # 1. Initialize the LlamaModel with bias
49 vllm_config.model_config.hf_config.bias = True
@@ -111,21 +124,9 @@ class TeleChat2ForCausalLM(LlamaForCausalLM):
124 def load_weights(self, weights: Iterable[Tuple[str,
125 torch.Tensor]]) -> Set[str]:
126
114 - hf_to_vllm_mapper = WeightsMapper(
115 - orig_to_new_prefix={
116 - "transformer.": "model.",
117 - },
118 - orig_to_new_substr={
119 - ".h.": ".layers.",
120 - ".self_attention.": ".self_attn.",
121 - ".word_embeddings.": ".embed_tokens.",
122 - ".dense.": ".o_proj.",
123 - ".ln_f.": ".norm.",
124 - },
125 - )
127 loader = AutoWeightsLoader(
128 self,
129 skip_prefixes=(["lm_head."]
130 if self.config.tie_word_embeddings else None),
131 )
131 - return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
132 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/ultravox.py
+4 -3
@@ -302,6 +302,9 @@ class ModifiedWhisperEncoder(WhisperEncoder):
302 @MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor)
303 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
304
305 + hf_to_vllm_mapper = WeightsMapper(
306 + orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
307 +
308 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
309 super().__init__()
310 config = vllm_config.model_config.hf_config
@@ -494,9 +497,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
497
498 def load_weights(self, weights: Iterable[Tuple[str,
499 torch.Tensor]]) -> Set[str]:
497 - hf_to_vllm_mapper = WeightsMapper(
498 - orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
500
501 loader = AutoWeightsLoader(self,
502 ignore_unexpected_prefixes=["audio_tower."])
502 - return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
503 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)