A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 13:33 UTC
Fix weight mapping test for Transfomers v5 (#33162)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jan 27, 2026 at 12:30 UTC
14385c80fc78c92e789b12fd4cbd1be2dc5df653
3 files changed
+11
-4
tests/models/multimodal/test_mapping.py
+6
-1
@@ -30,7 +30,12 @@ def create_dummy_model(repo: str, model_arch: str) -> PreTrainedModel:
30
model_cls: PreTrainedModel = getattr(transformers, model_arch)
31
config = AutoConfig.from_pretrained(repo)
32
with torch.device("meta"):
33
- return model_cls._from_config(config)
33
+ model = model_cls._from_config(config)
34
+ # TODO(hmellor): Remove this once Transformers has fixed tied weights on meta device
35
+ # https://github.com/huggingface/transformers/issues/43522
36
+ if getattr(config.get_text_config(), "tie_word_embeddings", False):
37
+ model.tie_weights()
38
+ return model
39
40
41
def model_architectures_for_test() -> list[str]:
vllm/model_executor/models/transformers/base.py
+2
-1
@@ -249,7 +249,8 @@ class Base(
249
# Layers before module list
250
for name in pp_plan[:module_list_idx]:
251
if self.pp_group.is_first_rank or (
252
- self.text_config.tie_word_embeddings and self.pp_group.is_last_rank
252
+ getattr(self.text_config, "tie_word_embeddings", False)
253
+ and self.pp_group.is_last_rank
254
):
255
continue
256
setattr(self.model, name, PPMissingLayer())
vllm/model_executor/models/transformers/causal.py
+3
-2
@@ -38,7 +38,8 @@ class CausalMixin(VllmModelForTextGeneration):
38
39
# Tell `Base.load_weights` to skip
40
# `lm_head` if the model has tied word embeddings
41
- if self.text_config.tie_word_embeddings:
41
+ tie_word_embeddings = getattr(self.text_config, "tie_word_embeddings", False)
42
+ if tie_word_embeddings:
43
self.skip_prefixes.append("lm_head.")
44
45
if self.pp_group.is_last_rank:
@@ -48,7 +49,7 @@ class CausalMixin(VllmModelForTextGeneration):
49
quant_config=self.quant_config,
50
prefix=maybe_prefix(prefix, "lm_head"),
51
)
51
- if self.text_config.tie_word_embeddings:
52
+ if tie_word_embeddings:
53
self.lm_head = self.lm_head.tie_weights(
54
self.model.get_input_embeddings()
55
)