A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 08:45 UTC
Fix `tie_word_embeddings` for multimodal models in Transformers v5 (#33359)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jan 30, 2026 at 03:37 UTC
80b918f2bdc627f9e4ed8c2684cbd40cc1a082ad
1 file changed
+24
vllm/config/vllm.py
+24
@@ -458,6 +458,30 @@ class VllmConfig:
458
hf_config.architectures = architectures
459
460
model_config = copy.deepcopy(self.model_config)
461
+
462
+ if (
463
+ model_config.is_multimodal_model
464
+ and hasattr(model_config.hf_config, "tie_word_embeddings")
465
+ and not hasattr(hf_config.get_text_config(), "tie_word_embeddings")
466
+ ):
467
+ # In Transformers v5, tie_word_embeddings belongs to the config of the class
468
+ # that can see both layers to be tied. For example:
469
+ #
470
+ # SomeVLModel:
471
+ # self.language_model = SomeLanguageModel()
472
+ # self.vision_model = SomeVisionModel()
473
+ #
474
+ # SomeVLModelForMultimodalLM:
475
+ # self.model = SomeVLModel()
476
+ # self.lm_head = nn.Linear()
477
+ #
478
+ # Therefore, tie_word_embeddings is defined in SomeVLModelForMultimodalLM's
479
+ # config and is not present in SomeVLModel's config. In vLLM, the lm_head
480
+ # belongs to the language_model, so we must ensure that tie_word_embeddings
481
+ # is set in the language_model's config.
482
+ tie_word_embeddings = model_config.hf_config.tie_word_embeddings
483
+ hf_config.get_text_config().tie_word_embeddings = tie_word_embeddings
484
+
485
model_config.hf_config = hf_config
486
model_config.model_arch_config = model_config.get_model_arch_config()
487