A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 00:45 UTC
Better weight tying check for multimodal models (#38035)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 25, 2026 at 12:07 UTC
1e673a43ce0197d0ff05041e862e5022c57c1e07
1 file changed
+30
-17
vllm/config/vllm.py
+30
-17
@@ -13,10 +13,12 @@ from dataclasses import is_dataclass
13
from datetime import datetime
14
from enum import IntEnum
15
from functools import lru_cache
16
+from importlib.metadata import version
17
from pathlib import Path
18
from typing import TYPE_CHECKING, Any, Literal, TypeVar, get_args
19
20
import torch
21
+from packaging.version import Version
22
from pydantic import ConfigDict, Field, model_validator
23
24
import vllm.envs as envs
@@ -550,26 +552,37 @@ class VllmConfig: # type: ignore[misc]
552
553
model_config = copy.deepcopy(self.model_config)
554
555
+ # In Transformers v5, tie_word_embeddings belongs to the config of the class
556
+ # that can see both layers to be tied. For example:
557
+ #
558
+ # SomeVLModel:
559
+ # self.language_model = SomeLanguageModel(SomeVLTextConfig)
560
+ # self.vision_model = SomeVisionModel(SomeVLVisionConfig)
561
+ #
562
+ # SomeVLModelForMultimodalLM:
563
+ # self.model = SomeVLModel(SomeVLConfig)
564
+ # self.lm_head = nn.Linear()
565
+ #
566
+ # Therefore, tie_word_embeddings is defined in SomeVLConfig and is not present
567
+ # in SomeVLTextConfig*. In vLLM, the lm_head belongs to the language_model, so
568
+ # we must ensure that tie_word_embeddings is set in the language_model's config.
569
+ #
570
+ # *For some models, SomeVLTextConfig may also have a tie_word_embeddings field.
571
+ # This is only the case if SomeVLTextConfig is also used for a text only version
572
+ # of the same model. For example:
573
+ #
574
+ # SomeVLModelForCausalLM:
575
+ # self.model = SomeLanguageModel(SomeVLTextConfig)
576
+ # self.lm_head = nn.Linear()
577
+ #
578
+ # Therefore, the presence of tie_word_embeddings in SomeVLTextConfig cannot
579
+ # be used as a signal for whether tie_word_embeddings should be copied from
580
+ # hf_config to the language_model config.
581
if (
554
- model_config.is_multimodal_model
582
+ Version(version("transformers")) >= Version("5.0.0")
583
+ and model_config.is_multimodal_model
584
and hasattr(model_config.hf_config, "tie_word_embeddings")
556
- and not hasattr(hf_config.get_text_config(), "tie_word_embeddings")
585
):
558
- # In Transformers v5, tie_word_embeddings belongs to the config of the class
559
- # that can see both layers to be tied. For example:
560
- #
561
- # SomeVLModel:
562
- # self.language_model = SomeLanguageModel()
563
- # self.vision_model = SomeVisionModel()
564
- #
565
- # SomeVLModelForMultimodalLM:
566
- # self.model = SomeVLModel()
567
- # self.lm_head = nn.Linear()
568
- #
569
- # Therefore, tie_word_embeddings is defined in SomeVLModelForMultimodalLM's
570
- # config and is not present in SomeVLModel's config. In vLLM, the lm_head
571
- # belongs to the language_model, so we must ensure that tie_word_embeddings
572
- # is set in the language_model's config.
586
tie_word_embeddings = model_config.hf_config.tie_word_embeddings
587
hf_config.get_text_config().tie_word_embeddings = tie_word_embeddings
588